diff --git a/Cargo.lock b/Cargo.lock index afe947361..b49997d52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,7 +448,6 @@ dependencies = [ name = "asap-stage1-pass1-per-computation-candidates" version = "0.1.0" dependencies = [ - "asap-accuracy", "asap-types", "thiserror 2.0.18", ] diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 940cd9c02..666477f7e 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -3,11 +3,11 @@ name = "asap-accuracy" version = "0.1.0" edition = "2021" -# The accuracy model shared by the planning stages: per-family estimators, -# guarantee composition and propagation through a DAG, sizing, budget -# split, the target check and the -# deployment's accuracy model and evidence traits. Depends only on -# `asap-types`, never on a stage crate, so every stage can use it. +# The accuracy rules shared by the planning stages: per-algorithm +# guarantees (the theoretical guarantee of θ), the target check, the +# deployment's measured accuracy, each operator's error contract and α^log +# of a logical edge. Depends only on `asap-types`, never on a stage crate, +# so every stage can use it. [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } diff --git a/crates/accuracy/src/budget.rs b/crates/accuracy/src/budget.rs deleted file mode 100644 index 65d552250..000000000 --- a/crates/accuracy/src/budget.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! Split an end-to-end accuracy target equally among the approximate inputs -//! of one composition (#669, part b). -//! -//! When `n` approximate values compose into one output, each is sized for an -//! equal share of the output's target, so the composed bound meets the -//! target: `ε_i = ε / n` for an additively composed metric (no slack), -//! `ε_i = 1 − (1 + ε)^{−1/n}` for a relative one, and `δ_i = δ / n` (the -//! union bound). Relative errors compose through products and quotients: -//! each input contributes a factor of at most `1 + ε_i` as a factor and -//! `1/(1 − ε_i)` as a divisor, so the true error of any mix stays within -//! `(1 − ε_i)^{−n} − 1 = ε`. The composed bound Stage 3 checks meets it for -//! one product or quotient of two inputs (a ratio's `(a + b)/(1 − b)`), not -//! for a nested quotient such as `x / (y / z)`, whose bound composes each -//! step's slack. A lone approximate denominator over an exact numerator is -//! sized for the whole target, so its quotient's bound `ε/(1 − ε)` misses -//! it. Only equal splits are made. -use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; - -/// The target of one of `approximate_inputs` approximate inputs that -/// compose into a value with `target`, split equally in `metric`'s -/// composition (see the module doc). `None` for an exact target or an ε -/// that is not positive and finite: no approximate input meets it. One -/// input (or none) gets `target` itself. -pub fn per_input_target( - target: &AccuracyTarget, - metric: AccuracyMetric, - approximate_inputs: usize, -) -> Option { - if target.is_exact() || !(target.max_error.is_finite() && target.max_error > 0.0) { - return None; - } - if approximate_inputs <= 1 { - return Some(target.clone()); - } - let fraction = 1.0 / approximate_inputs.max(1) as f64; - let max_error = match metric { - AccuracyMetric::Relative => 1.0 - (1.0 + target.max_error).powf(-fraction), - _ => target.max_error * fraction, - }; - Some(AccuracyTarget { - max_error, - failure_probability: target.failure_probability.map(|delta| delta * fraction), - ..target.clone() - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Additive metrics split ε and δ by `n`; the `n` shares add up to the - /// target. - #[test] - fn an_additive_target_splits_epsilon_and_delta_equally() { - let target = AccuracyTarget::max_error(0.1).failure_probability(0.02); - let share = per_input_target(&target, AccuracyMetric::ABSOLUTE, 2).unwrap(); - assert_eq!( - share, - AccuracyTarget::max_error(0.05).failure_probability(0.01) - ); - assert_eq!( - per_input_target(&target, AccuracyMetric::Rank, 1), - Some(target.clone()) - ); - assert_eq!( - per_input_target(&AccuracyTarget::max_error(0.1), AccuracyMetric::Rank, 2) - .unwrap() - .failure_probability, - None - ); - } - - /// A relative target splits so that a quotient meets it too: - /// `(1 − ε_i)^{−n} = 1 + ε`. For ε = 0.05 over two inputs each gets - /// 0.0241, and both a ratio `(a + b)/(1 − b)` and a product - /// `(1 + a)(1 + b) − 1` of them stay within 0.05; the product-only split - /// `(1 + ε)^{1/2} − 1 = 0.0247` would give the ratio 0.0507. - #[test] - fn a_relative_target_splits_so_a_quotient_meets_it() { - let target = AccuracyTarget::max_error(0.05); - let share = per_input_target(&target, AccuracyMetric::Relative, 2) - .unwrap() - .max_error; - assert!((share - (1.0 - 1.05f64.powf(-0.5))).abs() < 1e-12); - let ratio = (share + share) / (1.0 - share); - let product = (1.0 + share) * (1.0 + share) - 1.0; - assert!(ratio <= 0.05 && product <= 0.05, "{ratio} {product}"); - assert_eq!( - per_input_target(&target, AccuracyMetric::Relative, 1), - Some(target) - ); - } - - /// No approximate input meets an exact target. - #[test] - fn an_exact_target_has_no_share() { - assert_eq!( - per_input_target(&AccuracyTarget::exact(), AccuracyMetric::ABSOLUTE, 2), - None - ); - assert_eq!( - per_input_target(&AccuracyTarget::max_error(0.0), AccuracyMetric::ABSOLUTE, 2), - None - ); - } -} diff --git a/crates/accuracy/src/composition.rs b/crates/accuracy/src/composition.rs deleted file mode 100644 index 81b647234..000000000 --- a/crates/accuracy/src/composition.rs +++ /dev/null @@ -1,1349 +0,0 @@ -//! Registered propagation rules for computations over uncertain inputs. -//! Stage 1 composes each candidate's guarantees with them -//! (`per_computation_candidates`); a rule that needs a statistic leaves an -//! `Unknown` leaf named by one of the constants below, which Stage 3 binds. -use asap_types::accuracy::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, - GuaranteeSource, ProbabilityExpr, -}; - -#[cfg(test)] -use crate::requirements::meets; -use crate::PropagationStats; - -/// Rows an exact aggregation folds (per group): bounded by the series -/// count of the sources the query reads. -pub const INPUT_ROW_COUNT: &str = "input_row_count"; -/// Rows one group of a SQL fold over an approximate relation folds: not -/// bounded by the series count of the sources it reads, so plan selection -/// leaves it unbound. -pub(crate) const GROUP_ROW_COUNT: &str = "group_row_count"; -/// Whether every input value is non-negative. -pub(crate) const VALUES_NON_NEGATIVE: &str = "values_non_negative"; -/// The absolute bound of a product of two estimates, -/// `|x|·e_y + |y|·e_x + e_x·e_y`, given enforced operand domains. -pub const PRODUCT_OPERAND_DOMAINS: &str = "product_operand_domains"; -/// The absolute bound of a quotient with an approximate denominator, -/// `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))`, given enforced operand domains -/// with the denominator's bounded away from zero. -pub(crate) const QUOTIENT_OPERAND_DOMAINS: &str = "quotient_operand_domains"; -/// `‖f‖₁` of the stream a Count-Min bound is normalized by. -pub const STREAM_L1_NORM: &str = "stream_l1_norm"; -/// `‖f‖₂` of the stream a Count Sketch bound is normalized by. -pub(crate) const STREAM_L2_NORM: &str = "stream_l2_norm"; -/// Rows of one source's stream a summary folds: with a constant update -/// weight `c`, `‖f‖₁ = |c| · rows` and `‖f‖₂ ≤ ‖f‖₁`. -pub(crate) const STREAM_ROW_COUNT: &str = "stream_row_count"; - -/// Which stream norm bounds a Count-Min or Count Sketch estimate, given its -/// summary's update weight. -#[derive(Debug, Clone, PartialEq)] -pub enum StreamNorm { - /// The weight is the source's raw value column: the source's norm - /// (`source_norm_statistic`). - Source(String), - /// A constant weight (a count): `|weight|` times the source's row count. - ConstantWeight { weight: f64, source: String }, -} - -/// The statistic for `norm` ([`STREAM_L1_NORM`] or [`STREAM_L2_NORM`]) of -/// one source's stream, e.g. `stream_l1_norm[m]`. -pub(crate) fn source_norm_statistic(norm: &str, source: &str) -> String { - format!("{norm}[{source}]") -} - -/// Whether every value folded is non-negative, for values that are raw -/// samples of `metric` or plain sums of them, e.g. -/// `values_non_negative[m]`: an indicator, 1 when proven. Plan selection -/// binds it to 1 when the data workload declares `metric` a counter or -/// non-negative, and leaves it unbound otherwise. -pub(crate) fn raw_data_samples_sign_statistic(metric: &str) -> String { - format!("{VALUES_NON_NEGATIVE}[{metric}]") -} - -/// The metric a `raw_data_samples_sign_statistic` names. -pub fn parse_raw_data_samples_sign_statistic(statistic: &str) -> Option<&str> { - statistic - .strip_prefix(VALUES_NON_NEGATIVE)? - .strip_prefix('[')? - .strip_suffix(']') -} - -/// The norm and source a `source_norm_statistic` names. -pub fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &str)> { - [STREAM_L1_NORM, STREAM_L2_NORM] - .into_iter() - .find_map(|norm| { - let source = statistic - .strip_prefix(norm)? - .strip_prefix('[')? - .strip_suffix(']')?; - Some((norm, source)) - }) -} - -fn additive( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, -) -> AccuracyGuarantee { - let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); - terms.push(local.spec.error.clone()); - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - local.spec.accuracy_metric, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, rule), - ) -} - -/// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors -/// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all -/// cross-products so the expression DAG is exact for any input count. -fn multiplicative( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, -) -> AccuracyGuarantee { - let factors: Vec<&BoundExpr> = inputs - .iter() - .map(|g| &g.spec.error) - .chain(std::iter::once(&local.spec.error)) - .collect(); - // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. - let mut terms = Vec::new(); - for mask in 1..(1u32 << factors.len()) { - let subset: Vec = factors - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, b)| (*b).clone()) - .collect(); - terms.push(if subset.len() == 1 { - subset.into_iter().next().expect("one element") - } else { - BoundExpr::Product { factors: subset } - }); - } - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), - ) -} - -fn lipschitz( - op: &CompositionOperator, - constant: f64, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, -) -> AccuracyGuarantee { - let input = &inputs[0]; - let scaled = BoundExpr::Scaled { - factor: constant, - inner: Box::new(input.spec.error.clone()), - }; - let (bound, delta) = match local { - Some(local) => ( - BoundExpr::Sum { - terms: vec![scaled, local.spec.error.clone()], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - input.spec.failure_probability.clone(), - local.spec.failure_probability.clone(), - ], - }, - ), - None => (scaled, input.spec.failure_probability.clone()), - }; - let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - bound, - delta, - composed_provenance( - op, - inputs, - local.unwrap_or(&exact_local), - "lipschitz_union_bound", - ), - ) -} - -/// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The -/// planner composes one *per-value* child guarantee over an unknown -/// number of input rows, so both the bound and the union bound scale by -/// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is -/// not supplied. Each input's normalized bound is first converted to -/// absolute units via the statistic its metric is normalized by (also -/// unknown unless supplied); a `Rank` input has no such conversion. -fn exact_sum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, true, "exact_sum_relative"); - } - let mut terms = Vec::with_capacity(inputs.len()); - let mut deltas = Vec::with_capacity(inputs.len()); - let mut provenance = Vec::new(); - for (i, input) in inputs.iter().enumerate() { - let absolute = - absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: format!( - "input {i} carries a {:?} guarantee, which has no registered \ - conversion to an absolute value error", - input.spec.accuracy_metric - ), - })?; - if let BoundExpr::Product { factors } = &absolute { - for f in factors { - if let BoundExpr::Unknown { statistic } = f { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - } - } - } - terms.push(absolute); - deltas.push(input.spec.failure_probability.clone()); - } - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_sum_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Product { - factors: vec![count.clone(), BoundExpr::Sum { terms }], - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), - }, - provenance, - )) -} - -/// Exact arithmetic mean over values with absolute-error guarantees. -/// Averaging cannot amplify the largest absolute input error. The event -/// that every row respects its bound is still protected conservatively -/// by a union bound over the input row count. -fn exact_average( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, true, "exact_average_relative"); - } - if inputs - .iter() - .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact average requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_average_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// Exact `max`/`min` over approximate inputs of one shared metric: the -/// returned value's error is at most the largest input bound (order -/// statistics are monotone under a uniform perturbation), with -/// probability by the union bound over every input row. This bounds the -/// returned *value*; it does not identify the true winning key. -fn exact_extremum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, false, "exact_extremum_relative"); - } - let metric = inputs[0].spec.accuracy_metric; - if inputs - .iter() - .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact max/min requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_extremum_union_bound", - )); - Ok(AccuracyGuarantee::new( - metric, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// An exact sum, mean, max or min of values each within relative `α_i` -/// is within relative `max α_i`. A max or min holds whatever the signs; -/// a sum or mean only over non-negative values (no cancellation), so -/// with `needs_sign` and the sign unknown the bound is the symbolic -/// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound -/// over every input row. -fn relative_fold( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - needs_sign: bool, - rule: &str, -) -> Result { - if needs_sign && stats.values_non_negative == Some(false) { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "relative error of a sum over signed values is unbounded".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let bound = if needs_sign && stats.values_non_negative.is_none() { - match &stats.sign_statistic { - // The sign of one metric's samples: the bound, gated on it. - Some(statistic) => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - BoundExpr::Product { - factors: vec![ - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - BoundExpr::Unknown { - statistic: statistic.clone(), - }, - ], - } - } - None => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - } - } - } - } else { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - }; - provenance.extend(composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact aggregation"), - rule, - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - bound, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// Exact division of two relative-value estimates (an exact operand has -/// bound 0). If the numerator is within relative `a` and the denominator -/// within `b < 1`, the ratio is within -/// `max((1 + a)/(1 − b) − 1, 1 − (1 − a)/(1 + b)) = (a + b)/(1 − b)`, and -/// fails with the union bound of their failure probabilities. No operand -/// domain is needed: a relative bound below 1 keeps a zero denominator at -/// zero and a numerator's sign, so the true and estimated ratios are then -/// the same non-finite value. Both bounds must be constants: a symbolic one -/// has no rule. -/// Float64 overflow and underflow of the quotient are not bounded, as for a -/// product. -fn exact_division( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], -) -> Result { - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason, - }; - if inputs.len() != 2 - || inputs.iter().any(|input| { - input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() - }) - { - return Err(unsupported( - "division needs two relative-value or exact guarantees".into(), - )); - } - let Some(numerator) = inputs[0].spec.error.evaluate() else { - return Err(unsupported( - "numerator relative bound is unavailable".into(), - )); - }; - let Some(denominator) = inputs[1].spec.error.evaluate() else { - return Err(unsupported( - "denominator relative bound is unavailable".into(), - )); - }; - if !(numerator.is_finite() && numerator >= 0.0 && (0.0..1.0).contains(&denominator)) { - return Err(unsupported( - "division needs finite non-negative bounds and a denominator bound below one".into(), - )); - } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (numerator + denominator) / (1.0 - denominator), - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|input| input.spec.failure_probability.clone()) - .collect(), - }, - composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact division"), - "relative_division_union_bound", - ), - )) -} - -/// Whether `guarantee` is a sketch's distinct-count estimate. -fn estimates_distinct_count(guarantee: &AccuracyGuarantee) -> bool { - let cardinality = format!( - "{:?}", - asap_types::summary_type::SketchStatistic::Cardinality - ); - guarantee.provenance.iter().any(|source| { - matches!(source, GuaranteeSource::SketchEvaluation { query, .. } if *query == cardinality) - }) -} - -/// Every input carries a relative-value guarantee. -fn all_relative(inputs: &[AccuracyGuarantee]) -> bool { - !inputs.is_empty() - && inputs - .iter() - .all(|g| g.spec.accuracy_metric == AccuracyMetric::Relative) -} - -/// `stats.input_row_count` as a bound factor, or an `Unknown` leaf named by -/// `stats.row_count_statistic` (recorded in `provenance`) when absent. -fn row_count(stats: &PropagationStats, provenance: &mut Vec) -> BoundExpr { - match stats.input_row_count { - Some(n) => BoundExpr::Constant { value: n as f64 }, - None => { - let statistic = stats.row_count_statistic.unwrap_or(INPUT_ROW_COUNT); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }); - BoundExpr::Unknown { - statistic: statistic.into(), - } - } - } -} - -/// `input` with a Count-Min ([`AccuracyMetric::ABSOLUTE_STREAM_L1`]) or Count Sketch -/// ([`AccuracyMetric::ABSOLUTE_STREAM_L2`]) bound restated as an additive absolute -/// error: the normalized bound times its stream norm, a symbolic statistic -/// until bound. The norm is the one `norm` names ([`StreamNorm`]), which -/// plan selection can bind; without one (a weight computed from the -/// values) it is a statistic nothing binds. Any other guarantee is -/// returned unchanged. -pub(crate) fn additive_as_absolute( - input: AccuracyGuarantee, - norm: Option<&StreamNorm>, -) -> AccuracyGuarantee { - if !matches!( - input.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE_STREAM_L1 | AccuracyMetric::ABSOLUTE_STREAM_L2 - ) || input.is_exact() - { - return input; - } - let mut bound = absolute_bound(&input).expect("additive metrics convert to absolute errors"); - if let (Some(norm), BoundExpr::Product { factors }) = (norm, &mut bound) { - for factor in factors { - if let BoundExpr::Unknown { statistic } = factor { - *factor = match norm { - StreamNorm::Source(source) => BoundExpr::Unknown { - statistic: source_norm_statistic(statistic, source), - }, - StreamNorm::ConstantWeight { weight, source } => BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { - value: weight.abs(), - }, - BoundExpr::Unknown { - statistic: source_norm_statistic(STREAM_ROW_COUNT, source), - }, - ], - }, - }; - } - } - } - let mut provenance = input.provenance; - unknown_statistics(&bound, &mut |statistic| { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }) - }); - let mut absolute = AccuracyGuarantee { - provenance, - ..input - }; - absolute.spec.accuracy_metric = AccuracyMetric::ABSOLUTE; - absolute.spec.error = bound; - absolute -} - -/// Every `Unknown` statistic in `bound`. -fn unknown_statistics(bound: &BoundExpr, found: &mut dyn FnMut(&str)) { - match bound { - BoundExpr::Unknown { statistic } => found(statistic), - BoundExpr::Sum { terms } - | BoundExpr::Product { factors: terms } - | BoundExpr::Max { terms } => { - for term in terms { - unknown_statistics(term, found); - } - } - BoundExpr::Scaled { inner, .. } => unknown_statistics(inner, found), - _ => {} - } -} - -/// `input`'s bound converted to absolute value units, multiplying a -/// normalized metric by the (unknown) statistic it is normalized by. `None` -/// for a metric with no such conversion (`Rank`). -fn absolute_bound(input: &AccuracyGuarantee) -> Option { - let normalizer = match input.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => return Some(input.spec.error.clone()), - AccuracyMetric::Relative => "true_value_magnitude", - AccuracyMetric::ABSOLUTE_STREAM_L1 => STREAM_L1_NORM, - AccuracyMetric::ABSOLUTE_STREAM_L2 => STREAM_L2_NORM, - // `Rank` has no distribution-free conversion to a value error; a - // metric this crate does not know has no registered conversion. - AccuracyMetric::Rank | _ => return None, - }; - if input.spec.error.is_zero() { - return Some(BoundExpr::Zero); - } - Some(BoundExpr::Product { - factors: vec![ - input.spec.error.clone(), - BoundExpr::Unknown { - statistic: normalizer.into(), - }, - ], - }) -} - -fn composed_provenance( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, -) -> Vec { - let mut provenance: Vec = inputs - .iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - }) - .collect(); - provenance.extend(local.provenance.iter().cloned()); - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: rule.into(), - }); - provenance -} - -/// The built-in composition rule: `inputs`' guarantees (in the parent's -/// child order) composed with the parent's own `local` guarantee under `op`, -/// given the statistics in `stats` (all unknown in Stage 1; plan selection -/// binds some). `Err` is the fail-closed answer: no registered rule, or a -/// missing input guarantee. It means "no rule", not "infeasible": Pass 1 -/// keeps the candidate with `guarantee = None`. -pub(crate) fn propagate( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, -) -> Result { - // Exact input: only the local guarantee remains (or the value is exact). - if inputs.iter().all(AccuracyGuarantee::is_exact) - && !matches!(op, CompositionOperator::TopKSelection) - { - return Ok(match local { - Some(local) => { - let mut out = local.clone(); - out.provenance - .extend(inputs.iter().enumerate().map(|(input_index, g)| { - GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - } - })); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - None => { - let mut out = AccuracyGuarantee::exact(format!("{op:?} over exact inputs")); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - }); - } - - let input_metrics: Vec = - inputs.iter().map(|g| g.spec.accuracy_metric).collect(); - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: input_metrics.clone(), - local_metric: local.map(|g| g.spec.accuracy_metric), - reason, - }; - // An exact input is compatible with every metric; only approximate - // inputs constrain the rule. - let approximate: Vec<&AccuracyGuarantee> = inputs.iter().filter(|g| !g.is_exact()).collect(); - let same_metric = - |metric: AccuracyMetric| approximate.iter().all(|g| g.spec.accuracy_metric == metric); - - match op { - CompositionOperator::CheckedRelativeDivision => { - if inputs.len() != 2 || local.is_some() || !same_metric(AccuracyMetric::Relative) { - return Err(unsupported( - "checked division requires two exact/relative-value operands".into(), - )); - } - let a = inputs[0] - .spec - .error - .evaluate() - .ok_or_else(|| unsupported("unknown numerator bound".into()))?; - let b = inputs[1] - .spec - .error - .evaluate() - .ok_or_else(|| unsupported("unknown denominator bound".into()))?; - if !(0.0..1.0).contains(&b) || a < 0.0 || !a.is_finite() { - return Err(unsupported("invalid relative division bounds".into())); - } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (a + b) / (1.0 - b) + 4.0 * f64::EPSILON, - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }, - composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("checked floating-point division"), - "checked_relative_division_union_bound", - ), - )) - } - CompositionOperator::ApproximateAggregate => { - let local = local.ok_or_else(|| { - unsupported("approximate operator has no local guarantee to compose".into()) - })?; - if !same_metric(local.spec.accuracy_metric) { - return Err(unsupported(format!( - "no registered cross-metric rule from {input_metrics:?} to {:?}", - local.spec.accuracy_metric - ))); - } - // A distinct count over approximate values is not within the - // inputs' relative error: no rule, as for any summary whose - // estimate is not a value of its inputs. - if estimates_distinct_count(local) { - return Err(unsupported( - "no registered rule for a distinct count over approximate values".into(), - )); - } - match local.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => Ok(additive(op, inputs, local, "additive_union_bound")), - AccuracyMetric::Relative => { - if stats.values_non_negative == Some(false) { - return Err(unsupported( - "relative-error composition cannot use a known signed input".into(), - )); - } - if stats.values_non_negative.is_none() { - let mut provenance = composed_provenance( - op, - inputs, - local, - "relative_value_sign_unverified", - ); - provenance.extend(stats.evidence_provenance.clone()); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - return Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - }, - ProbabilityExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - }, - provenance, - )); - } - Ok(multiplicative(op, inputs, local)) - } - AccuracyMetric::Rank - | AccuracyMetric::ABSOLUTE_STREAM_L1 - | AccuracyMetric::ABSOLUTE_STREAM_L2 - | _ => Err(unsupported(format!( - "no registered same-metric composition rule for {:?} over {:?}", - local.spec.accuracy_metric, local.spec.accuracy_metric - ))), - } - } - CompositionOperator::Lipschitz { constant } => { - if !(constant.is_finite() && *constant >= 0.0) { - return Err(unsupported(format!( - "Lipschitz constant {constant} is not a finite non-negative number" - ))); - } - if inputs.len() != 1 || !same_metric(AccuracyMetric::ABSOLUTE) { - return Err(unsupported( - "Lipschitz rule is registered for exactly one absolute-error input".into(), - )); - } - if local.is_some_and(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) { - return Err(unsupported( - "Lipschitz rule needs an absolute-error local guarantee".into(), - )); - } - Ok(lipschitz(op, *constant, inputs, local)) - } - CompositionOperator::ExactSum => exact_sum(op, inputs, stats), - CompositionOperator::ExactAverage => exact_average(op, inputs, stats), - CompositionOperator::ExactExtremum => exact_extremum(op, inputs, stats), - CompositionOperator::ExactDivision => exact_division(op, inputs), - CompositionOperator::CounterRate - | CompositionOperator::InstantCounterRate - | CompositionOperator::CounterIncrease => Err(unsupported( - "counter reset detection and boundary extrapolation have no distribution-free \ - accuracy bound over approximate samples; exact samples remain exact" - .into(), - )), - CompositionOperator::TopKSelection => { - let selected = stats.topk_selected_lower_bound; - let excluded = stats.topk_excluded_upper_bound; - let delta = stats.topk_interval_failure_probability; - if selected.is_some_and(|value| !value.is_finite()) - || excluded.is_some_and(|value| !value.is_finite()) - || delta.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) - || selected - .zip(excluded) - .is_some_and(|(lower, upper)| lower <= upper) - { - return Err(unsupported( - "top-k confidence intervals overlap or contain invalid evidence".into(), - )); - } - let mut provenance = inputs - .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .collect::>(); - provenance.extend(stats.evidence_provenance.clone()); - if let Some(local) = local { - provenance.extend(local.provenance.clone()); - } - for (name, missing) in [ - ("topk_selected_lower_bound", selected.is_none()), - ("topk_excluded_upper_bound", excluded.is_none()), - ("topk_interval_failure_probability", delta.is_none()), - ] { - if missing { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: name.into(), - }); - } - } - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "topk_membership_margin_certificate".into(), - }); - let certified = selected.is_some() && excluded.is_some() && delta.is_some(); - // A certified margin makes the returned key set the true one, so - // each returned value keeps its score's error; without one, - // membership and hence the values are unknown. - let metric = approximate - .first() - .map_or(AccuracyMetric::ABSOLUTE, |score| score.spec.accuracy_metric); - if !same_metric(metric) { - return Err(unsupported( - "top-k selection needs scores of one metric".into(), - )); - } - let error = if certified { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - } else { - BoundExpr::Unknown { - statistic: "topk_membership_margin".into(), - } - }; - Ok(AccuracyGuarantee::new( - metric, - error, - delta.map_or_else( - || ProbabilityExpr::Unknown { - statistic: "topk_interval_failure_probability".into(), - }, - |value| ProbabilityExpr::Constant { value }, - ), - provenance, - )) - } - // An operator this crate does not know has no registered rule. - _ => Err(unsupported("no registered rule for this operator".into())), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A source-norm statistic names its norm and source, whatever - /// characters the source holds; any other statistic is not one. - #[test] - fn source_norm_statistics_round_trip() { - for source in ["m", "a[b]", "]x[", "t.c", ""] { - for norm in [STREAM_L1_NORM, STREAM_L2_NORM] { - let statistic = source_norm_statistic(norm, source); - assert_eq!( - parse_source_norm_statistic(&statistic), - Some((norm, source)) - ); - } - } - for other in [ - INPUT_ROW_COUNT, - STREAM_L1_NORM, - "stream_l3_norm[m]", - "stream_l1_norm[m", - ] { - assert_eq!(parse_source_norm_statistic(other), None, "{other}"); - } - } - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - fn rel(bound: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Zero, - vec![], - ) - } - fn with_metric(metric: AccuracyMetric, bound: f64) -> AccuracyGuarantee { - let mut g = abs(bound, 0.0); - g.spec.accuracy_metric = metric; - g - } - #[test] - fn checked_division_propagates_value_bounds_and_rejects_rank_bounds() { - let op = CompositionOperator::CheckedRelativeDivision; - let inputs = [rel(0.01), rel(0.01)]; - let g = propagate(&op, &inputs, None, &Default::default()).unwrap(); - assert!((g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-14); - let mut rank = inputs[0].clone(); - rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert!(propagate(&op, &[rank.clone(), rank], None, &Default::default()).is_err()); - assert!(propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()).is_err()); - } - /// A ratio of relative bounds `a`, `b` is within `(a + b)/(1 − b)`, - /// the larger of its over- and under-estimate bounds, with the union - /// bound on failure; no operand domain is needed. A denominator bound of - /// 1 or more, or a bound that is not finite and non-negative, has no - /// rule. - #[test] - fn relative_division_bounds_the_ratio_without_operand_domains() { - let mut numerator = rel(0.02); - numerator.spec.failure_probability = ProbabilityExpr::Constant { value: 0.001 }; - let got = propagate( - &CompositionOperator::ExactDivision, - &[numerator, rel(0.03)], - None, - &PropagationStats::default(), - ) - .unwrap(); - let (a, b): (f64, f64) = (0.02, 0.03); - let over = (1.0 + a) / (1.0 - b) - 1.0; - let under = 1.0 - (1.0 - a) / (1.0 + b); - assert!((got.spec.error.evaluate().unwrap() - over.max(under)).abs() < 1e-14); - assert!((got.spec.failure_probability.evaluate().unwrap() - 0.001).abs() < 1e-15); - for bound in [1., f64::NAN, f64::INFINITY, -0.1] { - assert!(propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(bound)], - None, - &PropagationStats::default() - ) - .is_err()); - } - let rank = with_metric(AccuracyMetric::Rank, 0.01); - assert!(propagate( - &CompositionOperator::ExactDivision, - &[rank.clone(), rank], - None, - &PropagationStats::default() - ) - .is_err()); - } - #[test] - fn exact_child_contributes_zero_error() { - let local = abs(0.05, 0.01); - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[AccuracyGuarantee::exact("sum")], - Some(&local), - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.spec.error.evaluate(), Some(0.05)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.01)); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - } - #[test] - fn additive_bounds_and_delta_union_bound_compose() { - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[abs(0.02, 0.01)], - Some(&abs(0.03, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.05).abs() < 1e-12); - // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::CompositionStep { rule, .. } if rule == "additive_union_bound" - ))); - } - #[test] - fn relative_error_includes_the_cross_term() { - let stats = PropagationStats { - values_non_negative: Some(true), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &stats, - ) - .unwrap(); - // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. - assert!((out.spec.error.evaluate().unwrap() - 0.32).abs() < 1e-12); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); - } - #[test] - fn relative_error_without_sign_knowledge_remains_symbolic() { - let unknown = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - let signed = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats { - values_non_negative: Some(false), - ..Default::default() - }, - ); - assert!(signed.is_err()); - } - #[test] - fn incompatible_metrics_are_rejected_not_treated_as_exact() { - // HLL cardinality error under a CMS frequency guarantee. - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Relative, 0.01)], - Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!( - err, - AccuracyError::UnsupportedComposition { - input_metrics, - local_metric: Some(AccuracyMetric::ABSOLUTE_STREAM_L1), - .. - } if input_metrics == vec![AccuracyMetric::Relative] - )); - // Quantile rank error under value-additive logic. - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&abs(0.01, 0.0)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn same_metric_rank_over_rank_has_no_registered_rule() { - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&with_metric(AccuracyMetric::Rank, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn lipschitz_scales_the_input_bound() { - let out = propagate( - &CompositionOperator::Lipschitz { constant: 3.0 }, - &[abs(0.1, 0.01)], - Some(&abs(0.05, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.35).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - } - #[test] - fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { - let out = propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!( - out.spec.error.evaluate(), - None, - "unknown row count stays unknown" - ); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" - ))); - assert!(!meets(&out, 1.0, None)); - - let known = PropagationStats { - input_row_count: Some(4), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.4).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - #[test] - fn exact_extremum_takes_the_max_bound() { - let known = PropagationStats { - input_row_count: Some(2), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ExactExtremum, - &[abs(0.1, 0.01), abs(0.3, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.3).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - /// `max`/`min` of values each within relative `α_i` is within relative - /// `max α_i` of the true extremum, whatever the signs. - #[test] - fn exact_extremum_keeps_a_relative_value_bound() { - let out = propagate( - &CompositionOperator::ExactExtremum, - &[rel(0.01), rel(0.02)], - None, - &PropagationStats { - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(out.spec.error.evaluate(), Some(0.02)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.0)); - } - /// A sum or mean of relative-error values keeps the relative bound only - /// over values known non-negative (cancellation otherwise); with the sign - /// unknown, the bound stays symbolic, and a known sign change rejects. - #[test] - fn exact_sum_and_average_keep_a_relative_bound_only_without_a_sign_change() { - for op in [ - CompositionOperator::ExactSum, - CompositionOperator::ExactAverage, - ] { - let with = |values_non_negative| { - propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - values_non_negative, - input_row_count: Some(3), - ..Default::default() - }, - ) - }; - let known = with(Some(true)).unwrap(); - assert_eq!(known.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(known.spec.error.evaluate(), Some(0.01), "{op:?}"); - let unknown = with(None).unwrap(); - assert_eq!(unknown.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!( - unknown.spec.error, - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into() - } - ); - assert!(with(Some(false)).is_err(), "{op:?}"); - } - } - /// Over the samples of one metric, an unknown sign is named by that - /// metric ([`raw_data_samples_sign_statistic`]) and gates the bound: binding it - /// to 1 (proven) gives the relative bound, and left unbound it stays - /// unknown. - #[test] - fn a_sign_over_one_metrics_samples_gates_the_relative_bound() { - let statistic = raw_data_samples_sign_statistic("m"); - assert_eq!(parse_raw_data_samples_sign_statistic(&statistic), Some("m")); - assert_eq!( - parse_raw_data_samples_sign_statistic(VALUES_NON_NEGATIVE), - None - ); - for op in [ - CompositionOperator::ExactSum, - CompositionOperator::ExactAverage, - ] { - let out = propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - sign_statistic: Some(statistic.clone()), - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.error.evaluate(), None, "{op:?}"); - let proven = out.bind(&|s| (s == statistic).then_some(1.0)); - assert_eq!(proven.spec.error.evaluate(), Some(0.01), "{op:?}"); - let other = out.bind(&|s| (s == raw_data_samples_sign_statistic("n")).then_some(1.0)); - assert_eq!(other.spec.error.evaluate(), None, "{op:?}"); - } - } - - #[test] - fn exact_average_has_its_own_absolute_error_rule() { - let out = propagate( - &CompositionOperator::ExactAverage, - &[abs(0.25, 0.01)], - None, - &PropagationStats { - input_row_count: Some(4), - ..PropagationStats::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(out.spec.error.evaluate(), Some(0.25)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.04)); - } - #[test] - fn topk_selection_requires_a_separated_margin_certificate() { - let unknown = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - - let certified = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(certified.spec.error.evaluate(), Some(0.1)); - assert_eq!(certified.spec.failure_probability.evaluate(), Some(0.005)); - - let overlapping = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(100.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ); - assert!(overlapping.is_err()); - - let partial = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert!(partial.has_unknown()); - assert_eq!(partial.spec.failure_probability.evaluate(), Some(0.005)); - - let invalid_partial = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(f64::NAN), - ..Default::default() - }, - ); - assert!(invalid_partial.is_err()); - } -} diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs new file mode 100644 index 000000000..95bd9399f --- /dev/null +++ b/crates/accuracy/src/error_contract/mod.rs @@ -0,0 +1,298 @@ +//! Each operator's error contract: the linear bound $g_c$ on a consumer's +//! output error from its inputs' errors (concepts doc, section 4). + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::operator::arguments::BinaryOpKind; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr}; +use asap_types::stages::{ColumnLineage, Memo, MemoGroupId}; +use asap_types::summary_type::ExactKind; + +/// $g_c$'s linear bound for errors in `metric`, each input's and the +/// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the +/// operator, its arguments and, for a `Filter`, its input group's column +/// lineage in `memo`. `None` when the consumer has no error contract for +/// `metric`: an edge from an input whose error is in `metric` then fails +/// (the metric rule). No factor is guessed; every contract keeps the metric +/// (concepts doc, section 4, "Local propagation"): +/// - value-preserving (`Sort`, `TimeRange`, `TimeShift`, `Concat`, a +/// readout, a finalize, a `SummaryMerge`): $\beta = 1$ in any metric; +/// - a `Project` whose items reading an estimate ([`reads_an_estimate`]) +/// each keep its error ([`keeps_error`]): $\beta = 1$, in any metric for +/// a copied column, in absolute error for `abs`, `clamp`, `clamp_min`, +/// `clamp_max` (also as the PromQL frontend's `promql_` functions) and a +/// negation, in relative error for a product with or quotient by a +/// nonzero literal (`q * 2`, `q / 1000`); +/// - a `Filter` whose predicate reads only columns copied from a source +/// (lineage `SourceColumn`, labels and keys): $\beta = 1$ in any metric; +/// - an `Aggregate` (or exact accumulator build) of counts ($\beta = 0$), +/// min and max (absolute or relative, $\beta = 1$) and avg (absolute, +/// $\beta = 1$), without measure filters or `HAVING`, whose group-by +/// keys are copied from a source as a `Filter`'s predicate columns are +/// (a count by an estimate has no bound); +/// - `+` and `-`: absolute, $\beta = 1$; `/`: relative, +/// $\beta = 1/(1-b)$ on both, the denominator capped at $b$; `*`: +/// relative, $(1+b, 1)$, the right operand capped at $b$ +/// ($(1+x)(1+y)-1 \le (1+b)\,x + y$ for $y \le b$), with $b$ = +/// [`RELATIVE_OPERAND_VALIDITY_CAP`]. +/// +/// Every other operator (a filter comparing a computed value, a ranking, +/// a `sum`, a comparison, another function) has none. +pub(crate) fn error_contract( + memo: &Memo, + consumer: &Operator, + metric: AccuracyMetric, +) -> Option { + let inputs = consumer.children().len(); + let uniform = |factor: f64| ErrorContract { + error_factor_per_input: vec![factor; inputs], + linear_bound_holds_up_to_per_input: vec![None; inputs], + }; + let absolute = matches!(metric, AccuracyMetric::Absolute { .. }); + let extremum_metric = absolute || metric == AccuracyMetric::Relative; + match consumer { + Operator::NonASAP(op) => match op { + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::Concat { .. } => Some(uniform(1.0)), + NonASAPOp::Project { cols, child } => cols + .iter() + .filter(|item| reads_an_estimate(memo, *child, &item.expr)) + .all(|item| keeps_error(&item.expr, metric)) + .then(|| uniform(1.0)), + NonASAPOp::Filter { pred, child } => (pred.0.operator_refs().is_empty() + && copied_from_a_source(memo, *child, pred.0.columns_referenced())) + .then(|| uniform(1.0)), + NonASAPOp::Aggregate { + measures, + filters, + having: None, + reduction, + child, + .. + } if filters.iter().all(Option::is_none) + && !measures.is_empty() + && keys_copied_from_a_source(memo, *child, reduction) => + { + let mut factor: f64 = 0.0; + for measure in measures { + match measure { + AggIntent::Count => {} + AggIntent::Min { .. } | AggIntent::Max { .. } if extremum_metric => { + factor = factor.max(1.0) + } + AggIntent::Avg { .. } if absolute => factor = factor.max(1.0), + _ => return None, + } + } + Some(uniform(factor)) + } + NonASAPOp::BinaryOp { + operator: + BinaryOperator { + kind: BinaryOpKind::Arithmetic(kind), + .. + }, + return_bool: false, + .. + } => { + let b = RELATIVE_OPERAND_VALIDITY_CAP; + match (kind, metric) { + (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, _) if absolute => { + Some(uniform(1.0)) + } + (ArithmeticOpKind::Div, AccuracyMetric::Relative) => Some(ErrorContract { + error_factor_per_input: vec![1.0 / (1.0 - b); 2], + linear_bound_holds_up_to_per_input: vec![None, Some(b)], + }), + (ArithmeticOpKind::Mul, AccuracyMetric::Relative) => Some(ErrorContract { + error_factor_per_input: vec![1.0 + b, 1.0], + linear_bound_holds_up_to_per_input: vec![None, Some(b)], + }), + _ => None, + } + } + _ => None, + }, + Operator::ASAP(op) => match op { + ASAPOp::SummaryEstimate { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::SummaryMerge { .. } => Some(uniform(1.0)), + ASAPOp::SummaryAgg { + summary_type: FieldDataType::ExactAggregate(kind, _), + filter: None, + reduction, + child, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: FieldDataType::ExactAggregate(kind, _), + filter: None, + reduction, + child, + .. + } if keys_copied_from_a_source(memo, *child, reduction) => match kind { + ExactKind::Count => Some(uniform(0.0)), + ExactKind::Min | ExactKind::Max if extremum_metric => Some(uniform(1.0)), + _ => None, + }, + _ => None, + }, + } +} + +/// Whether each of `columns` of `group`'s output is copied from a source +/// (lineage `SourceColumn`: a label or key), not computed: an estimate's +/// error can change which rows a predicate or a grouping on it selects. +fn copied_from_a_source( + memo: &Memo, + group: MemoGroupId, + columns: impl IntoIterator, +) -> bool { + let lineage = &memo.groups()[group.0 as usize] + .group_properties + .column_lineage; + columns + .into_iter() + .all(|column| matches!(lineage.get(column), Some(ColumnLineage::SourceColumn(_)))) +} + +/// Whether every key `reduction` groups `child`'s rows by is copied from a +/// source ([`copied_from_a_source`]): `by` keys as listed; a `without` +/// grouping keys by every column of `child` but the time, the sample value +/// and the excluded ones; a per-entity reduction keeps each row's group. +fn keys_copied_from_a_source(memo: &Memo, child: MemoGroupId, reduction: &Reduction) -> bool { + let Reduction::Reduce(by) = reduction else { + return true; + }; + if !by.is_without() { + return copied_from_a_source(memo, child, by.keys().iter().copied()); + } + let schema = &memo.groups()[child.0 as usize].group_properties.schema; + let value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); + copied_from_a_source( + memo, + child, + (0..schema.fields.len()).filter(|&column| { + schema.time_index != Some(column) + && value != Some(column) + && !by.keys().contains(&column) + }), + ) +} + +/// Whether `expr`, an item of a `Project` over `child`, reads a value that +/// may carry an estimate's error: a column of `child` that is neither +/// copied from a source (lineage `SourceColumn`: a label, a key, the series +/// identity) nor its time, or another operator's result. A PromQL function +/// of the value replaces only the value; the time and labels it copies +/// (and `promql_drop_metric_name` of the series identity) read no estimate. +fn reads_an_estimate(memo: &Memo, child: MemoGroupId, expr: &ScalarExpr) -> bool { + let properties = &memo.groups()[child.0 as usize].group_properties; + !expr.operator_refs().is_empty() + || expr.columns_referenced().into_iter().any(|column| { + properties.schema.time_index != Some(column) + && !matches!( + properties.column_lineage.get(column), + Some(ColumnLineage::SourceColumn(_)) + ) + }) +} + +/// Whether a `Project` item `expr` keeps the error in `metric` of the +/// column it reads, with $\beta = 1$: a copy of a column or a literal, or +/// the series identity without its metric name (`promql_drop_metric_name`, +/// which changes a label, not a value), in any metric; a 1-Lipschitz +/// function of a column ([`one_lipschitz`]) in absolute error; a column +/// times or divided by a nonzero literal in relative error. +fn keeps_error(expr: &ScalarExpr, metric: AccuracyMetric) -> bool { + match expr { + ScalarExpr::Column(_) | ScalarExpr::Literal(_) => true, + ScalarExpr::FunctionCall { name, args } if name == "promql_drop_metric_name" => { + matches!(args[..], [ScalarExpr::Column(_)]) + } + _ if matches!(metric, AccuracyMetric::Absolute { .. }) => one_lipschitz(expr), + _ if metric == AccuracyMetric::Relative => scaled_by_a_constant(expr), + _ => false, + } +} + +/// Whether `expr` changes a column's value by at most its error: `abs`, +/// `clamp`, `clamp_min` or `clamp_max` (or the PromQL frontend's +/// `promql_abs`, …) of a column with literal bounds, or a column's +/// negation (1-Lipschitz). +fn one_lipschitz(expr: &ScalarExpr) -> bool { + match expr { + ScalarExpr::Negative { expr, .. } => matches!(**expr, ScalarExpr::Column(_)), + ScalarExpr::FunctionCall { name, args } => { + let name = name.to_ascii_lowercase(); + matches!( + name.strip_prefix("promql_").unwrap_or(&name), + "abs" | "clamp" | "clamp_min" | "clamp_max" + ) && matches!(args.first(), Some(ScalarExpr::Column(_))) + && args[1..] + .iter() + .all(|arg| matches!(arg, ScalarExpr::Literal(_))) + } + _ => false, + } +} + +/// Whether `expr` is a column times a nonzero literal (either side) or +/// divided by one: $c\hat{x}$ is off from $cx$ by the same share as +/// $\hat{x}$ from $x$ (a unit conversion, `q * 1000`). +fn scaled_by_a_constant(expr: &ScalarExpr) -> bool { + let nonzero = |expr: &ScalarExpr| match expr { + ScalarExpr::Literal(ScalarValue::Float64(x)) => x.is_finite() && *x != 0.0, + ScalarExpr::Literal(ScalarValue::Int64(x)) => *x != 0, + _ => false, + }; + let column = |expr: &ScalarExpr| matches!(expr, ScalarExpr::Column(_)); + match expr { + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left, + right, + .. + } => (column(left) && nonzero(right)) || (nonzero(left) && column(right)), + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Div, + left, + right, + .. + } => column(left) && nonzero(right), + _ => false, + } +} + +/// $b$ of a ratio's denominator and a product's right operand: the +/// relative error under which [`error_contract`]'s linear bound holds, +/// $\varepsilon_{X/Y} \le (\varepsilon_X + \varepsilon_Y) / (1 - b)$ +/// (concepts doc, section 4). 0.1 is the weakest error of every +/// algorithm's list, so it never excludes a listed θ. +const RELATIVE_OPERAND_VALIDITY_CAP: f64 = 0.1; + +/// The error contract of one consumer in one metric: its output error is +/// at most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error +/// stays within $b_{c,i}$ (concepts doc, section 4). +#[derive(Debug, Clone)] +#[cfg_attr(test, derive(PartialEq))] +pub struct ErrorContract { + /// $\beta_{c,i}$ per input: how much of that input's error, in the + /// contract's metric, reaches the output. + pub error_factor_per_input: Vec, + /// $b_{c,i}$ per input: the largest error of that input up to which + /// the linear bound holds; `None` when it holds for any error (a ratio + /// bounds only its denominator's). + pub linear_bound_holds_up_to_per_input: Vec>, +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/error_contract/tests.rs b/crates/accuracy/src/error_contract/tests.rs new file mode 100644 index 000000000..cd46f06b0 --- /dev/null +++ b/crates/accuracy/src/error_contract/tests.rs @@ -0,0 +1,287 @@ +use super::*; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::scalar::CompareOpKind; +use asap_types::ir::schema::{DataType, PROMQL_SERIES_IDENTITY}; +use asap_types::ir::{ExprSemantics, Predicate, ProjectItem}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::test_support::memo_fixtures::{aggregate, arithmetic, compare, readout, scan}; +use asap_types::workload::DataWorkload; + +/// Constant error contracts. A `Filter` whose +/// predicate reads only columns copied from a source keeps or drops +/// whole rows: β = 1 in every metric. Over a readout of a KLL state of +/// `latency` by `job`, `job = "api"` reads such a column; `value > 0.5` +/// compares the computed estimate and has no contract. A `Project` of +/// `abs`, `clamp` or a negation of the value changes it by at most its +/// error: β = 1 for absolute errors only; one scaling it by a nonzero +/// constant keeps its relative error: β = 1 for relative errors. Only +/// the items reading the estimate are judged, so the series identity +/// `promql_drop_metric_name` computes does not count. +#[test] +fn key_filters_and_value_preserving_projects_have_unit_contracts() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let state = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: g1, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let readout = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.9 }, state)) + .unwrap(); + let lineage = &memo.groups()[readout.0 as usize] + .group_properties + .column_lineage; + let column_where = |copied: bool| { + lineage + .iter() + .position(|l| matches!(l, ColumnLineage::SourceColumn(_)) == copied) + .expect("the readout has a label and an estimate") + }; + let (job, estimate) = (column_where(true), column_where(false)); + let filter = |pred| { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child: readout, + }) + }; + let by_job = filter(compare( + job, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )); + let by_value = filter(compare( + estimate, + CompareOpKind::Gt, + ScalarValue::Float64(0.5), + ExprSemantics::Promql, + )); + for metric in AccuracyMetric::ALL { + assert_eq!( + error_contract(&memo, &by_job, metric).map(|contract| contract.error_factor_per_input), + Some(vec![1.0]), + "key-only filter, {metric:?}" + ); + assert_eq!( + error_contract(&memo, &by_value, metric), + None, + "a filter comparing the estimate, {metric:?}" + ); + } + // A `Project` over the readout, as the PromQL frontend lowers a + // function of the value: the time and every label copied, the series + // identity through `promql_drop_metric_name`, the estimate replaced. + let value = || Box::new(ScalarExpr::Column(estimate)); + let literal = |x: f64| ScalarExpr::Literal(ScalarValue::Float64(x)); + let project_value = |expr: ScalarExpr| { + let fields = &memo.groups()[readout.0 as usize] + .group_properties + .schema + .fields; + Operator::NonASAP(NonASAPOp::Project { + cols: fields + .iter() + .enumerate() + .map(|(column, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: if column == estimate { + expr.clone() + } else if field.name == PROMQL_SERIES_IDENTITY { + ScalarExpr::FunctionCall { + name: "promql_drop_metric_name".into(), + args: vec![ScalarExpr::Column(column)], + } + } else { + ScalarExpr::Column(column) + }, + }) + .collect(), + child: readout, + }) + }; + let factor = |project: &Operator, metric| { + error_contract(&memo, project, metric).map(|contract| contract.error_factor_per_input) + }; + let one_lipschitz = [ + ScalarExpr::FunctionCall { + name: "promql_abs".into(), + args: vec![*value()], + }, + ScalarExpr::FunctionCall { + name: "promql_clamp".into(), + args: vec![*value(), literal(0.0), literal(100.0)], + }, + ScalarExpr::FunctionCall { + name: "promql_clamp_max".into(), + args: vec![*value(), literal(5.0)], + }, + ScalarExpr::Negative { + expr: value(), + semantics: ExprSemantics::Promql, + }, + ]; + for expr in one_lipschitz { + let project = project_value(expr.clone()); + assert_eq!( + factor(&project, AccuracyMetric::ABSOLUTE), + Some(vec![1.0]), + "{expr:?}" + ); + assert_eq!(factor(&project, AccuracyMetric::Relative), None, "{expr:?}"); + } + // Scaling by a nonzero constant keeps a relative error: `q / 2`, + // `q * 2`, `2 * q`; scaling by 0 has no contract. + let scaled = |op, left, right| ScalarExpr::Arithmetic { + op, + left: Box::new(left), + right: Box::new(right), + semantics: ExprSemantics::Promql, + }; + for expr in [ + scaled(ArithmeticOpKind::Div, *value(), literal(2.0)), + scaled(ArithmeticOpKind::Mul, *value(), literal(2.0)), + scaled(ArithmeticOpKind::Mul, literal(2.0), *value()), + ] { + let project = project_value(expr.clone()); + assert_eq!( + factor(&project, AccuracyMetric::Relative), + Some(vec![1.0]), + "{expr:?}" + ); + assert_eq!(factor(&project, AccuracyMetric::Rank), None, "{expr:?}"); + } + let times_zero = project_value(scaled(ArithmeticOpKind::Mul, *value(), literal(0.0))); + assert_eq!(factor(×_zero, AccuracyMetric::Relative), None); + // Copying the value (and dropping the metric name from the series + // identity) keeps it in every metric. + let copied = project_value(*value()); + for metric in AccuracyMetric::ALL { + assert_eq!(factor(&copied, metric), Some(vec![1.0]), "{metric:?}"); + } +} + +/// A count (and min, max, avg) grouped by a column that is itself an +/// estimate has no contract: an estimate's error moves rows between +/// groups, so a count by it has no bound, as for a filter comparing it. +/// Over a readout of a KLL state of `latency` by `job`, `count by +/// (estimate)` and an exact count accumulator by it have none, while +/// `count by (job)` reads no value (factor 0). +#[test] +fn a_reduction_keyed_by_an_estimate_has_no_contract() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let state = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: g1, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let readout = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.99 }, state)) + .unwrap(); + let lineage = &memo.groups()[readout.0 as usize] + .group_properties + .column_lineage; + let column_where = |copied: bool| { + lineage + .iter() + .position(|l| matches!(l, ColumnLineage::SourceColumn(_)) == copied) + .expect("the readout has a label and an estimate") + }; + let (job, estimate) = (column_where(true), column_where(false)); + let exact_count_by = |key: usize| { + Operator::ASAP(ASAPOp::SummaryAgg { + child: readout, + summary_type: FieldDataType::exact(ExactKind::Count), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), + reduction: Reduction::by(vec![key]), + filter: None, + }) + }; + for metric in AccuracyMetric::ALL { + for measure in [ + AggIntent::Count, + AggIntent::Min { col: None }, + AggIntent::Max { col: None }, + AggIntent::Avg { col: None }, + ] { + assert_eq!( + error_contract( + &memo, + &aggregate(Reduction::by(vec![estimate]), measure.clone(), readout), + metric + ), + None, + "{measure:?} by the estimate, {metric:?}" + ); + } + assert_eq!( + error_contract(&memo, &exact_count_by(estimate), metric), + None, + "an exact count by the estimate, {metric:?}" + ); + for count_by_job in [ + aggregate(Reduction::by(vec![job]), AggIntent::Count, readout), + exact_count_by(job), + ] { + assert_eq!( + error_contract(&memo, &count_by_job, metric) + .map(|contract| contract.error_factor_per_input), + Some(vec![0.0]), + "a count by job, {metric:?}" + ); + } + } +} + +/// A ratio carries relative error with factor 1/(1 − 0.1) on both +/// inputs and caps its denominator at 0.1; a count reads no value +/// (factor 0); a sum has no contract. +#[test] +fn error_contracts_of_a_ratio_a_count_and_a_sum() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let divide = arithmetic(ArithmeticOpKind::Div, g1, g1); + assert_eq!( + error_contract(&memo, ÷, AccuracyMetric::Relative), + Some(ErrorContract { + error_factor_per_input: vec![1.0 / 0.9; 2], + linear_bound_holds_up_to_per_input: vec![None, Some(0.1)], + }) + ); + for metric in [AccuracyMetric::ABSOLUTE, AccuracyMetric::Rank] { + assert_eq!(error_contract(&memo, ÷, metric), None); + } + let by_job = || Reduction::by(vec![2]); + assert_eq!( + error_contract( + &memo, + &aggregate(by_job(), AggIntent::Count, g1), + AccuracyMetric::Rank + ) + .map(|contract| contract.error_factor_per_input), + Some(vec![0.0]) + ); + assert_eq!( + error_contract( + &memo, + &aggregate(by_job(), AggIntent::Sum { col: Some(1) }, g1), + AccuracyMetric::ABSOLUTE + ), + None + ); +} diff --git a/crates/accuracy/src/error_propagation.rs b/crates/accuracy/src/error_propagation.rs deleted file mode 100644 index d3d755f4e..000000000 --- a/crates/accuracy/src/error_propagation.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Each operator's local error propagation: the linear bound $g_c$ on a -//! consumer's output error from its inputs' errors (concepts doc, section 4). - -use asap_types::accuracy::AccuracyMetric; -use asap_types::ir::Operator; -use asap_types::stages::MemoGroupId; - -/// $g_c$'s linear bound for errors in `metric`, each input's and the -/// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the -/// operator and its arguments. `None` when the consumer has no error -/// contract for `metric`: an input whose error is in `metric` then gives -/// no edge (the metric rule). No factor is guessed. -pub fn error_propagation( - consumer: &Operator, - metric: AccuracyMetric, -) -> Option { - let _ = (consumer, metric); - todo!("#1207 PR 4: the error contract of each operator, per metric") -} - -/// The local error propagation of one consumer: its output error is at -/// most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error stays -/// within its validity cap (concepts doc, section 4; approved 2026-11-05). -#[derive(Debug, Clone, PartialEq)] -pub struct ErrorPropagation { - /// $\beta_{c,i}$ per input, for that input's metric. - pub factor_per_input: Vec, - /// $b_{c,i}$ per input: the cap under which the linear bound holds; - /// `None` when the bound holds everywhere (e.g. a ratio caps only its - /// denominator). - pub validity_cap_per_input: Vec>, -} diff --git a/crates/accuracy/src/function_rules.rs b/crates/accuracy/src/function_rules.rs deleted file mode 100644 index 1a91edea8..000000000 --- a/crates/accuracy/src/function_rules.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! The accuracy composition rule of each exact function applied to an -//! approximate input, shared by Pass 1's guarantee composition and the -//! accuracy algebra. -use asap_types::accuracy::CompositionOperator; -use asap_types::ir::operator::AggIntent; - -/// The composition operator that propagates an approximate input's guarantee -/// through `intent`; `None` for a function with no registered rule over -/// approximate input, whose callers must fail closed. -pub(crate) fn approximate_input_composition(intent: &AggIntent) -> Option { - Some(match intent { - AggIntent::Sum { .. } => CompositionOperator::ExactSum, - AggIntent::Min { .. } | AggIntent::Max { .. } => CompositionOperator::ExactExtremum, - AggIntent::Avg { .. } => CompositionOperator::ExactAverage, - AggIntent::Rate => CompositionOperator::CounterRate, - AggIntent::IRate => CompositionOperator::InstantCounterRate, - AggIntent::Increase => CompositionOperator::CounterIncrease, - _ => return None, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn counter_functions_have_distinct_definition_rules() { - assert_eq!( - approximate_input_composition(&AggIntent::Rate), - Some(CompositionOperator::CounterRate) - ); - assert_eq!( - approximate_input_composition(&AggIntent::IRate), - Some(CompositionOperator::InstantCounterRate) - ); - assert_eq!( - approximate_input_composition(&AggIntent::Increase), - Some(CompositionOperator::CounterIncrease) - ); - } -} diff --git a/crates/accuracy/src/guarantee/binomial.rs b/crates/accuracy/src/guarantee/binomial.rs new file mode 100644 index 000000000..d2d0b85b6 --- /dev/null +++ b/crates/accuracy/src/guarantee/binomial.rs @@ -0,0 +1,26 @@ +//! The binomial tail the per-row failure bounds share: a structure whose +//! independent rows each miss with probability `p` misses when at least +//! `k` of its `n` rows do. + +/// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded +/// up by a relative 1e-12 so rounding never certifies a smaller tail. +pub(super) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { + if k == 0 || p >= 1.0 { + return 1.0; + } + if k > n || p <= 0.0 { + return 0.0; + } + let (ln_p, ln_q) = (p.ln(), (-p).ln_1p()); + let mut ln_choose = 0.0; + let mut tail = 0.0; + for i in 0..=n { + if i > 0 { + ln_choose += f64::from(n - i + 1).ln() - f64::from(i).ln(); + } + if i >= k { + tail += (ln_choose + f64::from(i) * ln_p + f64::from(n - i) * ln_q).exp(); + } + } + (tail * (1.0 + 1e-12)).min(1.0) +} diff --git a/crates/accuracy/src/guarantee/cardinality.rs b/crates/accuracy/src/guarantee/cardinality/mod.rs similarity index 50% rename from crates/accuracy/src/guarantee/cardinality.rs rename to crates/accuracy/src/guarantee/cardinality/mod.rs index 620d93742..9d64e6285 100644 --- a/crates/accuracy/src/guarantee/cardinality.rs +++ b/crates/accuracy/src/guarantee/cardinality/mod.rs @@ -1,6 +1,6 @@ //! KMV's provable distinct-count bound. //! -//! # KMV (#694) +//! # KMV //! //! `asap_sketchlib`'s KMV keeps the `k` smallest distinct 64-bit hashes of //! its items and estimates (`src/sketches/kmv.rs`): @@ -25,44 +25,49 @@ //! //! Neither bound depends on `N`, so no distinct bound is needed: the //! failure probability ([`kmv_failure_probability`]) is their sum, per -//! output row (each group's estimate). Sizing (test-only -//! `sizing_oracle::kmv_k`) takes the smallest `k` with that sum at most -//! `δ`, about -//! `1 + (1+ε)(2+ε)·ln(2/δ)/ε²` (below `3·ln(2/δ)/ε² + 1` for `ε ≤ 1`). +//! output row (each group's estimate). The guarantee at a failure +//! probability δ ([`guarantee`]) is the smallest `ε` with that sum at +//! most δ. use super::*; /// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. -pub(super) const KMV_MIN_K: u32 = 2; +const KMV_MIN_K: u32 = 2; +/// The smallest relative error `k` misses with probability at most +/// `failure_probability` ([`kmv_failure_probability`], by bisection); +/// `None` when no error up to 1 does. pub(super) fn guarantee( - algorithm: &SketchAlgorithm, params: &SketchParams, - query: &SketchStatistic, + failure_probability: f64, ) -> Option { - match params { - SketchParams::Kmv { k } => { - // Its relative standard error. - let eps = 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(); - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::Relative, - eps, - ProbabilityExpr::Constant { - value: kmv_failure_probability(*k, eps), - }, - "kmv_exact_below_k_chernoff_v1", - )) + let SketchParams::Kmv { k } = params else { + return None; + }; + if kmv_failure_probability(*k, 1.0) > failure_probability { + return None; + } + let (mut met, mut missed) = (1.0, 0.0); + for _ in 0..60 { + let eps = (met + missed) / 2.0; + if kmv_failure_probability(*k, eps) <= failure_probability { + met = eps; + } else { + missed = eps; } - _ => None, } + Some(super::bounded_guarantee( + AccuracyMetric::Relative, + met, + ProbabilityExpr::Constant { + value: failure_probability, + }, + )) } /// The probability that `asap_sketchlib`'s KMV with `k` misses relative /// error `eps` on any number of distinct items (the module's bound): `1` /// when it certifies nothing. -pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { +fn kmv_failure_probability(k: u32, eps: f64) -> f64 { if k < KMV_MIN_K || !(eps.is_finite() && eps > 0.0) { return 1.0; } @@ -78,35 +83,4 @@ pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { } #[cfg(test)] -mod tests { - use super::*; - - /// The bound dominates the exact failure probability of the order - /// statistic for every number of distinct items checked: exact below - /// `k`, and both binomial tails from `k` on. - #[test] - fn kmv_bound_dominates_the_exact_order_statistic_tails() { - use super::super::binomial_upper_tail; - for k in [16u32, 64, 256] { - for eps in [0.1, 0.3, 0.6] { - let bound = kmv_failure_probability(k, eps); - for n in (k..8 * k).step_by(7).chain([k, 20 * k]) { - let nf = f64::from(n); - let ranks = f64::from(k - 1); - let over = binomial_upper_tail(n, ranks / ((1.0 + eps) * nf), k); - let under_p = ranks / ((1.0 - eps) * nf); - let under = if under_p >= 1.0 { - 0.0 - } else { - 1.0 - binomial_upper_tail(n, under_p, k) - }; - assert!( - over + under <= bound + 1e-9, - "k={k} eps={eps} n={n}: {}>{bound}", - over + under - ); - } - } - } - } -} +mod tests; diff --git a/crates/accuracy/src/guarantee/cardinality/tests.rs b/crates/accuracy/src/guarantee/cardinality/tests.rs new file mode 100644 index 000000000..e656e2055 --- /dev/null +++ b/crates/accuracy/src/guarantee/cardinality/tests.rs @@ -0,0 +1,30 @@ +use super::*; + +/// The bound dominates the exact failure probability of the order +/// statistic for every number of distinct items checked: exact below +/// `k`, and both binomial tails from `k` on. +#[test] +fn kmv_bound_dominates_the_exact_order_statistic_tails() { + use super::super::binomial_upper_tail; + for k in [16u32, 64, 256] { + for eps in [0.1, 0.3, 0.6] { + let bound = kmv_failure_probability(k, eps); + for n in (k..8 * k).step_by(7).chain([k, 20 * k]) { + let nf = f64::from(n); + let ranks = f64::from(k - 1); + let over = binomial_upper_tail(n, ranks / ((1.0 + eps) * nf), k); + let under_p = ranks / ((1.0 - eps) * nf); + let under = if under_p >= 1.0 { + 0.0 + } else { + 1.0 - binomial_upper_tail(n, under_p, k) + }; + assert!( + over + under <= bound + 1e-9, + "k={k} eps={eps} n={n}: {}>{bound}", + over + under + ); + } + } + } +} diff --git a/crates/accuracy/src/guarantee/cms.rs b/crates/accuracy/src/guarantee/cms.rs deleted file mode 100644 index 7333ea698..000000000 --- a/crates/accuracy/src/guarantee/cms.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Count-Min Sketch L1 frequency error. -use super::*; - -pub(super) fn guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { - let (SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. }) = - params - else { - return None; - }; - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::ABSOLUTE_STREAM_L1, - std::f64::consts::E / f64::from(*width), - ProbabilityExpr::Constant { - value: (-f64::from(*depth)).exp(), - }, - "count_min_l1_markov_v1", - )) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn local_guarantee_inverts_frequency_sizing() { - use crate::guarantee::sizing_oracle::size_params; - let c = asap_types::ir::operator::agg_intent::default_cardinality(); - let params = size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = sketch_guarantee( - &SketchAlgorithm::Cms, - ¶ms, - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(meets(&g, 0.01, Some(0.001))); - } - - #[test] - fn heap_evaluation_retains_frequency_metric() { - let cms_heap = SketchParams::CmsWithHeap { - width: 272, - depth: 5, - heap_size: 10, - }; - let topk_frequency = sketch_guarantee( - &SketchAlgorithm::CmsWithHeap, - &cms_heap, - &SketchStatistic::TopK { k: 10 }, - ) - .expect("heap sketch still provides per-key frequency intervals"); - assert_eq!( - topk_frequency.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE_STREAM_L1 - ); - } -} diff --git a/crates/accuracy/src/guarantee/cms/mod.rs b/crates/accuracy/src/guarantee/cms/mod.rs new file mode 100644 index 000000000..91063d661 --- /dev/null +++ b/crates/accuracy/src/guarantee/cms/mod.rs @@ -0,0 +1,20 @@ +//! Count-Min Sketch L1 frequency error. +use super::*; + +pub(super) fn guarantee(params: &SketchParams) -> Option { + let (SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. }) = + params + else { + return None; + }; + Some(super::bounded_guarantee( + AccuracyMetric::ABSOLUTE_STREAM_L1, + std::f64::consts::E / f64::from(*width), + ProbabilityExpr::Constant { + value: (-f64::from(*depth)).exp(), + }, + )) +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/cms/tests.rs b/crates/accuracy/src/guarantee/cms/tests.rs new file mode 100644 index 000000000..12b35605f --- /dev/null +++ b/crates/accuracy/src/guarantee/cms/tests.rs @@ -0,0 +1,16 @@ +use super::*; + +#[test] +fn heap_evaluation_retains_frequency_metric() { + let cms_heap = SketchParams::CmsWithHeap { + width: 272, + depth: 5, + heap_size: 10, + }; + let topk_frequency = guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0) + .expect("heap sketch still provides per-key frequency intervals"); + assert_eq!( + topk_frequency.spec.accuracy_metric, + AccuracyMetric::ABSOLUTE_STREAM_L1 + ); +} diff --git a/crates/accuracy/src/guarantee/count_sketch.rs b/crates/accuracy/src/guarantee/count_sketch.rs deleted file mode 100644 index 9c6ec7011..000000000 --- a/crates/accuracy/src/guarantee/count_sketch.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! CountSketch L2 frequency error and odd-depth median concentration: the -//! median fails only when a majority of rows fail, an exact binomial tail. -use super::*; - -pub(super) fn guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { - let (SketchParams::CountSketch { width, depth } - | SketchParams::CountSketchWithHeap { width, depth, .. }) = params - else { - return None; - }; - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::ABSOLUTE_STREAM_L2, - (3.0 / f64::from(*width)).sqrt(), - ProbabilityExpr::Constant { - value: count_sketch_failure_probability(*width, *depth)?, - }, - "count_sketch_l2_median_binomial_v1", - )) -} -/// One row misses a point query by more than `ε ‖f‖₂ = √(3/w) ‖f‖₂` with -/// probability at most `q/ε² = q w/3` (Chebyshev, `Var ≤ q F2`), with `q` -/// the column collision probability ([`column_collision`]): `1/3` to -/// within `w/(3 · 2^32)`. -pub(super) fn row_failure_probability(width: u32) -> f64 { - column_collision(width) * f64::from(width.max(1)) / 3.0 -} - -/// `P(c_i = c_j)` for two keys in a row of `cols` columns, at most -/// `1/cols + 2^-32`. The executor builds `CountSketch<_, RegularPath>` and -/// `CountSketchWithHeap<_, RegularPath>`, whose rows take a key's column as -/// `(hash & (2^32 − 1)) % cols`: each column holds `⌊2^32/cols⌋` or -/// `⌈2^32/cols⌉` of the `2^32` hash values, so a key lands in any one with -/// probability at most `1/cols + 2^-32`, and two keys collide no more -/// often than that. -fn column_collision(cols: u32) -> f64 { - 1.0 / f64::from(cols.max(1)) + 2f64.powi(-32) -} - -/// The median of an odd `depth` of independent rows misses only when at -/// least `(depth + 1)/2` rows miss: `P(Binomial(depth, p) ≥ (depth+1)/2)`. -pub(super) fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { - if depth == 0 || depth.is_multiple_of(2) { - return None; - } - Some(super::binomial_upper_tail( - depth, - row_failure_probability(width), - depth.div_ceil(2), - )) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Measured on the library's own `CountSketch<_, RegularPath>`: two - /// keys share a column of a 1536-wide row about `1/1536` of the time. - /// A `⌈log2 w⌉`-bit hash slice modulo `w` would give `1.125/1536`, 6 - /// standard deviations above; the bound used here is `1/w + 2^-32`. - #[test] - fn library_rows_collide_at_one_over_the_width() { - use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; - let width = 1536; - let mut row = CountSketch::, RegularPath>::with_dimensions(1, width); - row.update(&DataInput::U64(u64::MAX)); - let probes = 4_000_000u64; - let collisions = (0..probes) - .filter(|&key| row.estimate(&DataInput::U64(key)) != 0.0) - .count() as f64; - let expected = probes as f64 * column_collision(width as u32); - let sigma = expected.sqrt(); - assert!( - (collisions - expected).abs() < 4.0 * sigma, - "{collisions} vs {expected}" - ); - } - - /// Seeded end to end: the sized sketch (ε = 0.05, δ = 0.01, width 1200, - /// depth 47) answers each key of a skewed stream within `ε ‖f‖₂`, at - /// most a δ share of keys missing. - #[test] - fn sized_sketch_meets_its_guarantee_on_a_seeded_stream() { - use crate::guarantee::sizing_oracle::{count_sketch_depth, count_sketch_width}; - use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; - let (eps, delta) = (0.05, 0.01); - let width = count_sketch_width(eps); - let depth = count_sketch_depth(width, delta); - let mut sketch = CountSketch::, RegularPath>::with_dimensions( - depth as usize, - width as usize, - ); - // 20 000 keys, key i weighted 1e6 / (i + 1) (Zipf, s = 1), in a - // seeded order. - let keys = 20_000u64; - let weight = |key: u64| (1e6 / (key + 1) as f64).round(); - let mut state = 0x9E37_79B9_7F4A_7C15u64; - let mut order: Vec = (0..keys).collect(); - for i in (1..order.len()).rev() { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - order.swap(i, (state % (i as u64 + 1)) as usize); - } - for &key in &order { - sketch.update_weighted(&DataInput::U64(key), weight(key)); - } - let l2 = (0..keys).map(|k| weight(k).powi(2)).sum::().sqrt(); - let bound = (3.0 / f64::from(width)).sqrt() * l2; - let misses = (0..keys) - .filter(|&k| (sketch.estimate(&DataInput::U64(k)) - weight(k)).abs() > bound) - .count(); - assert!(misses as f64 <= delta * keys as f64, "{misses} misses"); - } - - #[test] - fn count_sketch_uses_an_l2_guarantee() { - use crate::guarantee::sizing_oracle::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - let intent = default_cardinality(); - let count_sketch = size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = sketch_guarantee( - &SketchAlgorithm::CountSketch, - &count_sketch, - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: Some("1".into()), - }, - ) - .expect("CountSketch has a parameter-derived L2 guarantee"); - assert_eq!( - guarantee.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE_STREAM_L2 - ); - assert!(meets(&guarantee, 0.01, Some(0.01))); - } -} diff --git a/crates/accuracy/src/guarantee/count_sketch/mod.rs b/crates/accuracy/src/guarantee/count_sketch/mod.rs new file mode 100644 index 000000000..57df2d2b7 --- /dev/null +++ b/crates/accuracy/src/guarantee/count_sketch/mod.rs @@ -0,0 +1,52 @@ +//! CountSketch L2 frequency error and odd-depth median concentration: the +//! median fails only when a majority of rows fail, an exact binomial tail. +use super::*; + +pub(super) fn guarantee(params: &SketchParams) -> Option { + let (SketchParams::CountSketch { width, depth } + | SketchParams::CountSketchWithHeap { width, depth, .. }) = params + else { + return None; + }; + Some(super::bounded_guarantee( + AccuracyMetric::ABSOLUTE_STREAM_L2, + (3.0 / f64::from(*width)).sqrt(), + ProbabilityExpr::Constant { + value: count_sketch_failure_probability(*width, *depth)?, + }, + )) +} +/// One row misses a point query by more than `ε ‖f‖₂ = √(3/w) ‖f‖₂` with +/// probability at most `q/ε² = q w/3` (Chebyshev, `Var ≤ q F2`), with `q` +/// the column collision probability ([`column_collision`]): `1/3` to +/// within `w/(3 · 2^32)`. +fn row_failure_probability(width: u32) -> f64 { + column_collision(width) * f64::from(width.max(1)) / 3.0 +} + +/// `P(c_i = c_j)` for two keys in a row of `cols` columns, at most +/// `1/cols + 2^-32`. The executor builds `CountSketch<_, RegularPath>` and +/// `CountSketchWithHeap<_, RegularPath>`, whose rows take a key's column as +/// `(hash & (2^32 − 1)) % cols`: each column holds `⌊2^32/cols⌋` or +/// `⌈2^32/cols⌉` of the `2^32` hash values, so a key lands in any one with +/// probability at most `1/cols + 2^-32`, and two keys collide no more +/// often than that. +fn column_collision(cols: u32) -> f64 { + 1.0 / f64::from(cols.max(1)) + 2f64.powi(-32) +} + +/// The median of an odd `depth` of independent rows misses only when at +/// least `(depth + 1)/2` rows miss: `P(Binomial(depth, p) ≥ (depth+1)/2)`. +fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { + if depth == 0 || depth.is_multiple_of(2) { + return None; + } + Some(super::binomial_upper_tail( + depth, + row_failure_probability(width), + depth.div_ceil(2), + )) +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/count_sketch/tests.rs b/crates/accuracy/src/guarantee/count_sketch/tests.rs new file mode 100644 index 000000000..f0505631f --- /dev/null +++ b/crates/accuracy/src/guarantee/count_sketch/tests.rs @@ -0,0 +1,55 @@ +use super::*; + +/// Measured on the library's own `CountSketch<_, RegularPath>`: two +/// keys share a column of a 1536-wide row about `1/1536` of the time. +/// A `⌈log2 w⌉`-bit hash slice modulo `w` would give `1.125/1536`, 6 +/// standard deviations above; the bound used here is `1/w + 2^-32`. +#[test] +fn library_rows_collide_at_one_over_the_width() { + use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; + let width = 1536; + let mut row = CountSketch::, RegularPath>::with_dimensions(1, width); + row.update(&DataInput::U64(u64::MAX)); + let probes = 4_000_000u64; + let collisions = (0..probes) + .filter(|&key| row.estimate(&DataInput::U64(key)) != 0.0) + .count() as f64; + let expected = probes as f64 * column_collision(width as u32); + let sigma = expected.sqrt(); + assert!( + (collisions - expected).abs() < 4.0 * sigma, + "{collisions} vs {expected}" + ); +} + +#[test] +fn count_sketch_uses_an_l2_guarantee() { + // ε = √(3/30 000) = 0.01; 47 rows fail together with probability ≤ 0.01. + let count_sketch = SketchParams::CountSketch { + width: 30_000, + depth: 47, + }; + let guarantee = guarantee_of_parameters( + &count_sketch, + &SketchStatistic::PointCount { + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, + value: Some("1".into()), + }, + 1.0, + ) + .expect("CountSketch has a parameter-derived L2 guarantee"); + assert_eq!( + guarantee.spec.accuracy_metric, + AccuracyMetric::ABSOLUTE_STREAM_L2 + ); + assert!(guarantee + .spec + .error + .evaluate() + .is_some_and(|error| within_limit(error, 0.01))); + assert!(guarantee + .spec + .failure_probability + .evaluate() + .is_some_and(|failure_probability| within_limit(failure_probability, 0.01))); +} diff --git a/crates/accuracy/src/guarantee/ddsketch.rs b/crates/accuracy/src/guarantee/ddsketch.rs index 9184de771..fde3d390f 100644 --- a/crates/accuracy/src/guarantee/ddsketch.rs +++ b/crates/accuracy/src/guarantee/ddsketch.rs @@ -1,21 +1,13 @@ //! DDSketch relative value error under its estimator/domain contract. use super::*; -pub(super) fn guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { +pub(super) fn guarantee(params: &SketchParams) -> Option { let SketchParams::DDSketch { alpha } = params else { return None; }; Some(super::bounded_guarantee( - algorithm, - params, - query, AccuracyMetric::Relative, *alpha, ProbabilityExpr::Zero, - "ddsketch_relative_error_alpha_v1", )) } diff --git a/crates/accuracy/src/guarantee/hll.rs b/crates/accuracy/src/guarantee/hll.rs index fc483ec2b..ef83b3e67 100644 --- a/crates/accuracy/src/guarantee/hll.rs +++ b/crates/accuracy/src/guarantee/hll.rs @@ -1,374 +1,20 @@ -//! Estimator-specific confidence for classic HLL's linear-counting branch. -//! -//! This is conditional on independent uniform bucket hashes and an enforced -//! upper bound on distinct items in the complete evaluation population (including -//! all merged tumbling windows). It is not an RSE-to-normal conversion or an ERP fit. +//! HLL's relative standard error, without a failure probability. use super::*; -pub(super) fn generic_guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { +pub(super) fn guarantee(params: &SketchParams) -> Option { let SketchParams::Hll { precision, .. } = params else { return None; }; Some(super::bounded_guarantee( - algorithm, - params, - query, AccuracyMetric::Relative, relative_standard_error(*precision), ProbabilityExpr::Unknown { statistic: "hll_estimator_failure_probability".into(), }, - "generic_hll_rse_only_no_confidence_v1", )) } /// HLL's relative standard error, `1.04 / √m` for `m = 2^precision` buckets. fn relative_standard_error(precision: u8) -> f64 { 1.04 / 2f64.powi(i32::from(precision)).sqrt() } - -/// Names classic HLL's failure probability at `precision` and a relative -/// error as a function of `max_distinct[source]`, the deployment's bound on -/// distinct items per evaluation of `source`, which plan selection binds. -const CLASSIC_FAILURE_PROBABILITY: &str = "classic_hll_failure_probability"; -const MAX_DISTINCT: &str = "max_distinct"; - -/// The relative error a classic HLL's guarantee states: the one its -/// precision was sized for, else its relative standard error. -pub fn classic_relative_error(precision: u8, relative_error: Option) -> f64 { - relative_error.unwrap_or_else(|| relative_standard_error(precision)) -} - -/// The statistic classic HLL's failure probability at `precision` and -/// `relative_error` over `source` is left as (`classic_guarantee`). -/// `relative_error` is written in Rust's shortest round-trip form, so it -/// parses back to the same value. -pub fn classic_failure_statistic(precision: u8, relative_error: f64, source: &str) -> String { - format!( - "{CLASSIC_FAILURE_PROBABILITY}[p={precision},eps={relative_error}]({MAX_DISTINCT}[{source}])" - ) -} - -/// The precision, relative error and source a classic HLL failure -/// statistic (`classic_failure_statistic`) names. -pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str)> { - let rest = statistic - .strip_prefix(CLASSIC_FAILURE_PROBABILITY)? - .strip_prefix("[p=")?; - let (params, rest) = rest.split_once(']')?; - let (precision, relative_error) = params.split_once(",eps=")?; - let source = rest - .strip_prefix('(')? - .strip_prefix(MAX_DISTINCT)? - .strip_prefix('[')? - .strip_suffix("])")?; - Some(( - precision.parse().ok()?, - relative_error.parse().ok()?, - source, - )) -} - -/// Classic HLL's guarantee for a distinct count over `source`'s own rows -/// (one instance per group, over the source's scan at most behind filters -/// and windows): relative error at most the one its precision was sized for -/// ([`classic_relative_error`]), with a failure probability -/// (`ClassicHllConfidence`) at that error left symbolic in the source's -/// distinct bound ([`classic_failure_statistic`]). The failure probability -/// is per output row: each group's estimate fails with at most it. `None` -/// for a precision outside the contract's. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn classic_guarantee( - precision: u8, - relative_error: Option, - source: &str, -) -> Option { - if !(4..=MAX_PRECISION).contains(&precision) { - return None; - } - let relative_error = classic_relative_error(precision, relative_error); - let statistic = classic_failure_statistic(precision, relative_error, source); - Some(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: relative_error, - }, - ProbabilityExpr::Unknown { - statistic: statistic.clone(), - }, - vec![GuaranteeSource::SketchEvaluation { - algorithm: "Hll".into(), - contract: CLASSIC_CONTRACT.into(), - params: serde_json::json!({"precision": precision, - "max_distinct": format!("{MAX_DISTINCT}[{source}]"), - "relative_error": relative_error, - "hash_assumption": "independent_uniform_buckets", - "population_scope": "complete_evaluation_including_merged_tumbling_windows"}), - query: "Cardinality".into(), - }], - )) -} - -/// The failure probability a classic HLL failure statistic -/// (`classic_failure_statistic`) names, given the -/// source's distinct bound: `1` (no certificate) when the bound is outside -/// the contract's range for this precision and relative error. -#[cfg(test)] -pub(crate) fn classic_failure_probability( - precision: u8, - relative_error: f64, - max_distinct: u64, -) -> f64 { - u32::try_from(max_distinct) - .ok() - .and_then(|n| ClassicHllConfidence::new(n, relative_error)) - .and_then(|model| model.failure_probability(precision)) - .unwrap_or(1.0) -} - -/// The largest precision classic HLL's contract covers. -pub const MAX_PRECISION: u8 = 18; - -const CLASSIC_CONTRACT: &str = "classic_hll_linear_counting_collision_bound_v1"; - -/// The largest distinct bound the contract supports at any precision: -/// `m/2` at precision 18. Evaluating a bound is O(bound). -#[cfg(test)] -pub(super) const MAX_CONTRACT_DISTINCT: u32 = 1 << 17; - -/// A finite-population contract for `m * ln(m / zero_registers)` with the -/// classic HLL small-range switch. Hashing is assumed independent and uniform. -/// The deployment must establish the population bound; observations alone do -/// not establish it. Unsupported precisions/populations return no certificate. -#[cfg(test)] -#[derive(Debug, Clone, Copy)] -pub(crate) struct ClassicHllConfidence { - max_distinct: u32, - relative_error: f64, -} - -#[cfg(test)] -impl ClassicHllConfidence { - pub(crate) fn new(max_distinct: u32, relative_error: f64) -> Option { - (max_distinct > 0 - && max_distinct <= MAX_CONTRACT_DISTINCT - && relative_error.is_finite() - && (1e-6..1.0).contains(&relative_error)) - .then_some(Self { - max_distinct, - relative_error, - }) - } - - /// Finite bound, not an asymptotic RSE fit. With N distinct hashes and K - /// occupied buckets, C=N-K collision arrivals satisfy - /// P(C>=t) <= lambda^t/t!, lambda=N(N-1)/(2m): each arrival's conditional - /// collision probability is at most (i-1)/m, and a union bound over t - /// arrivals is bounded by the t-th power of their sum divided by t!. - /// - /// N<=m/2 makes the classic raw estimate <=2*alpha_m*m<2.5m, - /// so the small-range switch always uses L=-m*ln(1-K/m). Then - /// K<=L<=N + N^2/(2(m-N)). The latter bounds overestimation - /// deterministically; underestimation implies C>epsilon*N. - /// We maximize the collision bound over EVERY integer N in the contract, - /// not just its upper endpoint (small-cardinality tails matter). - pub(super) fn failure_probability(&self, precision: u8) -> Option { - if !(4..=MAX_PRECISION).contains(&precision) { - return None; - } - let m = f64::from(1u32 << precision); - let max_n = f64::from(self.max_distinct); - // Reserve numerical slack; do not certify sub-floating-point error. - let eps = self.relative_error - 1e-8; - if max_n > m / 2.0 || max_n / (2.0 * (m - max_n)) > eps { - return None; - } - let mut log_factorial = vec![0.0; self.max_distinct as usize + 1]; - for i in 1..log_factorial.len() { - log_factorial[i] = log_factorial[i - 1] + (i as f64).ln(); - } - let mut worst = 0.0_f64; - for n in 2..=self.max_distinct { - let nf = f64::from(n); - // Including a boundary collision event is conservative. - let t = ((eps * nf).floor() as usize + 1).min(n as usize); - let lambda = nf * (nf - 1.0) / (2.0 * m); - let log_tail = (t as f64) * lambda.ln() - log_factorial[t]; - worst = worst.max(log_tail.min(0.0).exp()); - } - // Never return a spurious zero from underflow or numeric cancellation. - Some((worst * (1.0 + 1e-10) + 1e-12).min(1.0)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The failure-probability statistic names its precision, relative - /// error and source, any source name. - #[test] - fn classic_failure_statistics_round_trip() { - for source in ["flows_src_ip", "a[b]", "x])(", ""] { - for eps in [0.02, 0.1 + 0.2, 1.04 / 64.0] { - let statistic = classic_failure_statistic(12, eps, source); - assert_eq!( - parse_classic_failure_statistic(&statistic), - Some((12, eps, source)) - ); - } - } - for other in [ - "max_distinct[m]", - "classic_hll_failure_probability[p=x,eps=0.02](max_distinct[m])", - "classic_hll_failure_probability[p=12](max_distinct[m])", - ] { - assert_eq!(parse_classic_failure_statistic(other), None); - } - } - - /// The symbolic guarantee states the relative error the precision was - /// sized for and binds to the model's probability at that error; an - /// out-of-range bound certifies nothing. - #[test] - fn classic_guarantee_binds_the_distinct_bound() { - let guarantee = classic_guarantee(17, Some(0.02), "m").unwrap(); - assert_eq!(guarantee.spec.error.evaluate(), Some(0.02)); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - let expected = ClassicHllConfidence::new(1_000, 0.02) - .unwrap() - .failure_probability(17) - .unwrap(); - let bound = guarantee.bind(&|statistic| { - let (precision, eps, source) = parse_classic_failure_statistic(statistic)?; - (source == "m").then(|| classic_failure_probability(precision, eps, 1_000)) - }); - assert_eq!(bound.spec.failure_probability.evaluate(), Some(expected)); - assert!(expected <= 0.01); - for too_large in [u64::from(MAX_CONTRACT_DISTINCT) + 1, u64::MAX] { - assert_eq!(classic_failure_probability(18, 0.05, too_large), 1.0); - } - // Not sized for a target: its relative standard error. - let unsized_guarantee = classic_guarantee(12, None, "m").unwrap(); - assert_eq!( - unsized_guarantee.spec.error.evaluate(), - Some(relative_standard_error(12)) - ); - assert!(classic_guarantee(3, Some(0.02), "m").is_none()); - } - - /// The largest distinct bound certified at δ = 0.01 for a target ε, by - /// precision (#694's table): past the former 4096 computation cap at - /// precision 18. - #[test] - fn largest_certified_bounds_by_precision() { - for (eps, precision, largest) in [ - (0.05, 15, 1176), - (0.05, 18, 9636), - (0.02, 17, 1842), - (0.02, 18, 3791), - (0.01, 18, 72), - ] { - assert!( - classic_failure_probability(precision, eps, largest) <= 0.01, - "eps={eps} p={precision} n={largest}" - ); - assert!( - classic_failure_probability(precision, eps, largest + 1) > 0.01, - "eps={eps} p={precision} n={}", - largest + 1 - ); - } - } - - /// Exact occupancy probabilities independently check both tails for every N. - #[test] - fn probability_bound_dominates_exact_occupancy_distribution() { - for precision in 4..=10 { - let m = 1usize << precision; - let max_n = 64.min(m / 2); - for eps in [0.05, 0.2, 0.6] { - let model = ClassicHllConfidence::new(max_n as u32, eps).unwrap(); - let Some(bound) = model.failure_probability(precision) else { - continue; - }; - let mut occupancy = vec![0.0; max_n + 1]; - occupancy[0] = 1.0; - for n in 1..=max_n { - let mut next = vec![0.0; max_n + 1]; - for k in 0..n { - next[k] += occupancy[k] * k as f64 / m as f64; - next[k + 1] += occupancy[k] * (m - k) as f64 / m as f64; - } - occupancy = next; - let actual: f64 = occupancy - .iter() - .enumerate() - .filter_map(|(k, &prob)| { - let estimate = -(m as f64) * (-(k as f64) / (m as f64)).ln_1p(); - ((estimate - n as f64).abs() > eps * n as f64).then_some(prob) - }) - .sum(); - assert!( - actual <= bound + 1e-12, - "p={precision} n={n} eps={eps}: {actual}>{bound}" - ); - } - } - } - } - /// The model's evaluation formula matches the actual classic estimator after merge. - #[test] - fn native_classic_estimator_and_merged_registers_use_the_same_contract() { - use asap_sketchlib::sketches::hll::{Classic, HyperLogLogP16}; - let model = ClassicHllConfidence::new(128, 0.05).unwrap(); - assert!(model.failure_probability(16).unwrap() < 0.01); - let mut single = HyperLogLogP16::::new(); - let mut left = HyperLogLogP16::::new(); - let mut right = HyperLogLogP16::::new(); - for n in 0..128u64 { - // SplitMix64 supplies deterministic test hashes, not a proof of randomness. - let mut h = n.wrapping_add(0x9e3779b97f4a7c15); - h = (h ^ (h >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); - h = (h ^ (h >> 27)).wrapping_mul(0x94d049bb133111eb); - h ^= h >> 31; - single.update_with_hash(h); - if n % 2 == 0 { - left.update_with_hash(h); - } else { - right.update_with_hash(h); - } - } - left.merge(&right); - assert_eq!(single.registers_as_slice(), left.registers_as_slice()); - let zeroes = left - .registers_as_slice() - .iter() - .filter(|&&r| r == 0) - .count(); - let expected = (65536.0 * (65536.0 / zeroes as f64).ln()) as usize; - assert_eq!(left.estimate(), expected); - assert!((expected as f64 - 128.0).abs() / 128.0 <= 0.05); - } - #[test] - fn generic_rse_sizing_does_not_certify_confidence() { - use crate::guarantee::sizing_oracle::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - let c = default_cardinality(); - let params = size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = sketch_guarantee( - &SketchAlgorithm::Hll, - ¶ms, - &SketchStatistic::Cardinality, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(meets(&g, 0.01, None)); - assert!(!meets(&g, 0.01, Some(0.01))); - } -} diff --git a/crates/accuracy/src/guarantee/kll.rs b/crates/accuracy/src/guarantee/kll.rs deleted file mode 100644 index 0c509da0f..000000000 --- a/crates/accuracy/src/guarantee/kll.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! KLL normalized rank error at the registered empirical 99% calibration -//! and, below a 0.01 failure probability, that calibration scaled by the -//! Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`. -use super::*; -use asap_types::summary_type::KLL_MAX_K; - -pub(super) fn guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { - let SketchParams::Kll { k } = params else { - return None; - }; - if *k > KLL_MAX_K { - return None; - } - let delta = KLL_CALIBRATED_FAILURE_PROBABILITY; - let scale = delta_scale(delta)?; - let contract = if delta < KLL_CALIBRATED_FAILURE_PROBABILITY { - KLL_DELTA_SCALED_CONTRACT - } else { - KLL_EMPIRICAL_99_CONTRACT - }; - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::Rank, - kll_rank_error_99(*k) * scale, - ProbabilityExpr::Constant { value: delta }, - contract, - )) -} -/// The empirical 99% calibration of the pinned Apache DataSketches KLL. -const KLL_EMPIRICAL_99_CONTRACT: &str = "apache_datasketches_kll_empirical_99_a9b42755072b"; -/// That calibration scaled to a failure probability below 0.01 by the -/// Gaussian quantile ratio ([`delta_scale`]): a model, not a measurement, so -/// it is named apart from the calibration. -const KLL_DELTA_SCALED_CONTRACT: &str = - "apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b"; -pub(crate) const KLL_RANK_ERROR_COEFFICIENT_99: f64 = 2.296; -pub(crate) const KLL_RANK_ERROR_EXPONENT_99: f64 = 0.9723; -/// The failure probability of the empirical 99% calibration. -pub(crate) const KLL_CALIBRATED_FAILURE_PROBABILITY: f64 = 0.01; - -pub(crate) fn kll_rank_error_99(k: u32) -> f64 { - KLL_RANK_ERROR_COEFFICIENT_99 / f64::from(k).powf(KLL_RANK_ERROR_EXPONENT_99) -} - -/// How much the rank error at failure probability `delta` exceeds the error -/// at the calibrated 0.01, modelling the error as Gaussian: the ratio of the -/// two-sided standard normal quantiles, `z(1 − delta/2) / z(1 − 0.01/2)`. -/// It grows faster as `delta` shrinks than the KLL bound's -/// `sqrt(ln(1/delta))` (1.277 vs 1.225 at `delta` = 0.001), so it is the -/// conservative choice. 1 for `delta >= 0.01`; `None` unless `0 < delta < 1`. -pub(super) fn delta_scale(delta: f64) -> Option { - if !(delta > 0.0 && delta < 1.0) { - None - } else if delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { - Some(1.0) - } else { - // `z(1 − p) = −z(p)`, computed from the lower tail: `1 − delta/2` - // loses `delta` to rounding once it nears f64's epsilon. - Some( - normal_quantile(delta / 2.0) - / normal_quantile(KLL_CALIBRATED_FAILURE_PROBABILITY / 2.0), - ) - } -} - -/// The standard normal quantile `z(p)` for `0 < p < 1`, by Acklam's rational -/// approximation (relative error below 1.2e-9). -fn normal_quantile(p: f64) -> f64 { - const A: [f64; 6] = [ - -3.969_683_028_665_376e1, - 2.209_460_984_245_205e2, - -2.759_285_104_469_687e2, - 1.383_577_518_672_69e2, - -3.066_479_806_614_716e1, - 2.506_628_277_459_239, - ]; - const B: [f64; 5] = [ - -5.447_609_879_822_406e1, - 1.615_858_368_580_409e2, - -1.556_989_798_598_866e2, - 6.680_131_188_771_972e1, - -1.328_068_155_288_572e1, - ]; - const C: [f64; 6] = [ - -7.784_894_002_430_293e-3, - -3.223_964_580_411_365e-1, - -2.400_758_277_161_838, - -2.549_732_539_343_734, - 4.374_664_141_464_968, - 2.938_163_982_698_783, - ]; - const D: [f64; 4] = [ - 7.784_695_709_041_462e-3, - 3.224_671_290_700_398e-1, - 2.445_134_137_142_996, - 3.754_408_661_907_416, - ]; - let tail = |q: f64| { - (((((C[0] * q + C[1]) * q + C[2]) * q + C[3]) * q + C[4]) * q + C[5]) - / ((((D[0] * q + D[1]) * q + D[2]) * q + D[3]) * q + 1.0) - }; - const P_LOW: f64 = 0.024_25; - if p < P_LOW { - tail((-2.0 * p.ln()).sqrt()) - } else if p > 1.0 - P_LOW { - -tail((-2.0 * (1.0 - p).ln()).sqrt()) - } else { - let q = p - 0.5; - let r = q * q; - (((((A[0] * r + A[1]) * r + A[2]) * r + A[3]) * r + A[4]) * r + A[5]) * q - / (((((B[0] * r + B[1]) * r + B[2]) * r + B[3]) * r + B[4]) * r + 1.0) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn local_guarantee_inverts_rank_sizing() { - use crate::guarantee::sizing_oracle::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - let q = default_quantile(0.99); - let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = sketch_guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(meets(&g, 0.01, None)); - assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(meets(&g, 0.01, Some(0.01))); - assert_eq!(g.approximate_layer_count(), 1); - assert!(g.provenance.iter().any(|source| matches!( - source, - GuaranteeSource::SketchEvaluation { contract, .. } - if contract == "apache_datasketches_kll_empirical_99_a9b42755072b" - ))); - } - - /// A hand-built `k` above the library's cap gives no guarantee. - #[test] - fn a_k_above_the_library_cap_gives_no_guarantee() { - let g = |k| { - guarantee( - &SketchAlgorithm::Kll, - &SketchParams::Kll { k }, - &SketchStatistic::Quantile { q: 0.99 }, - ) - }; - assert!(g(KLL_MAX_K).is_some()); - assert!(g(KLL_MAX_K + 1).is_none()); - } - - /// Below δ = 0.01 the error scales by the Gaussian quantile ratio - /// `z(1 − δ/2) / z(1 − 0.01/2)`. - #[test] - fn delta_scale_is_the_gaussian_quantile_ratio() { - use crate::guarantee::sizing_oracle::{kll_k, kll_params}; - assert_eq!(delta_scale(0.01), Some(1.0)); - assert_eq!(delta_scale(0.5), Some(1.0)); - let ratio = delta_scale(0.001).unwrap(); - assert!((ratio - 1.277_463_039_557_138_7).abs() < 1e-8, "{ratio}"); - assert!((normal_quantile(0.975) - 1.959_963_984_540_054).abs() < 1e-8); - // Far in the tail, `1 − δ/2` rounds to 1 (or near it) in f64, so the - // scale must be computed from the lower tail, `−z(δ/2)`. - for (delta, expected) in [ - (1e-16, 3.224_121_029_197_093_4), - (1e-15, 3.116_223_140_825_122_4), - ] { - let scale = delta_scale(delta).unwrap(); - assert!((scale - expected).abs() < 1e-8, "{delta}: {scale}"); - } - assert_eq!( - kll_params(0.01, 0.001), - SketchParams::Kll { - k: kll_k(0.01 / ratio), - } - ); - } -} diff --git a/crates/accuracy/src/guarantee/kll/mod.rs b/crates/accuracy/src/guarantee/kll/mod.rs new file mode 100644 index 000000000..1280e756a --- /dev/null +++ b/crates/accuracy/src/guarantee/kll/mod.rs @@ -0,0 +1,112 @@ +//! KLL normalized rank error at the registered empirical 99% calibration +//! and, below a 0.01 failure probability, that calibration scaled by the +//! Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`. +use super::*; +use asap_types::summary_type::KLL_MAX_K; + +/// The rank error of `k` at failure probability `failure_probability`: the +/// calibration at 0.01 and above, scaled by [`delta_scale`] below. `None` +/// for a `k` above the library's cap or a probability outside (0, 1]. +pub(super) fn guarantee( + params: &SketchParams, + failure_probability: f64, +) -> Option { + let SketchParams::Kll { k } = params else { + return None; + }; + if *k > KLL_MAX_K || !(failure_probability > 0.0 && failure_probability <= 1.0) { + return None; + } + let delta = failure_probability.min(KLL_CALIBRATED_FAILURE_PROBABILITY); + let scale = delta_scale(delta)?; + Some(super::bounded_guarantee( + AccuracyMetric::Rank, + kll_rank_error_99(*k) * scale, + ProbabilityExpr::Constant { value: delta }, + )) +} +/// The empirical 99% calibration of the pinned Apache DataSketches KLL: +/// rank error `2.296 / k^0.9723` at failure probability 0.01. Below 0.01 it +/// is scaled by the Gaussian quantile ratio ([`delta_scale`]), a model, not +/// a measurement. +const KLL_RANK_ERROR_COEFFICIENT_99: f64 = 2.296; +const KLL_RANK_ERROR_EXPONENT_99: f64 = 0.9723; +/// The failure probability of the empirical 99% calibration. +const KLL_CALIBRATED_FAILURE_PROBABILITY: f64 = 0.01; + +fn kll_rank_error_99(k: u32) -> f64 { + KLL_RANK_ERROR_COEFFICIENT_99 / f64::from(k).powf(KLL_RANK_ERROR_EXPONENT_99) +} + +/// How much the rank error at failure probability `delta` exceeds the error +/// at the calibrated 0.01, modelling the error as Gaussian: the ratio of the +/// two-sided standard normal quantiles, `z(1 − delta/2) / z(1 − 0.01/2)`. +/// It grows faster as `delta` shrinks than the KLL bound's +/// `sqrt(ln(1/delta))` (1.277 vs 1.225 at `delta` = 0.001), so it is the +/// conservative choice. 1 for `delta >= 0.01`; `None` unless `0 < delta < 1`. +fn delta_scale(delta: f64) -> Option { + if !(delta > 0.0 && delta < 1.0) { + None + } else if delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { + Some(1.0) + } else { + // `z(1 − p) = −z(p)`, computed from the lower tail: `1 − delta/2` + // loses `delta` to rounding once it nears f64's epsilon. + Some( + normal_quantile(delta / 2.0) + / normal_quantile(KLL_CALIBRATED_FAILURE_PROBABILITY / 2.0), + ) + } +} + +/// The standard normal quantile `z(p)` for `0 < p < 1`, by Acklam's rational +/// approximation (relative error below 1.2e-9). +fn normal_quantile(p: f64) -> f64 { + const A: [f64; 6] = [ + -3.969_683_028_665_376e1, + 2.209_460_984_245_205e2, + -2.759_285_104_469_687e2, + 1.383_577_518_672_69e2, + -3.066_479_806_614_716e1, + 2.506_628_277_459_239, + ]; + const B: [f64; 5] = [ + -5.447_609_879_822_406e1, + 1.615_858_368_580_409e2, + -1.556_989_798_598_866e2, + 6.680_131_188_771_972e1, + -1.328_068_155_288_572e1, + ]; + const C: [f64; 6] = [ + -7.784_894_002_430_293e-3, + -3.223_964_580_411_365e-1, + -2.400_758_277_161_838, + -2.549_732_539_343_734, + 4.374_664_141_464_968, + 2.938_163_982_698_783, + ]; + const D: [f64; 4] = [ + 7.784_695_709_041_462e-3, + 3.224_671_290_700_398e-1, + 2.445_134_137_142_996, + 3.754_408_661_907_416, + ]; + let tail = |q: f64| { + (((((C[0] * q + C[1]) * q + C[2]) * q + C[3]) * q + C[4]) * q + C[5]) + / ((((D[0] * q + D[1]) * q + D[2]) * q + D[3]) * q + 1.0) + }; + const P_LOW: f64 = 0.024_25; + if p < P_LOW { + tail((-2.0 * p.ln()).sqrt()) + } else if p > 1.0 - P_LOW { + -tail((-2.0 * (1.0 - p).ln()).sqrt()) + } else { + let q = p - 0.5; + let r = q * q; + (((((A[0] * r + A[1]) * r + A[2]) * r + A[3]) * r + A[4]) * r + A[5]) * q + / (((((B[0] * r + B[1]) * r + B[2]) * r + B[3]) * r + B[4]) * r + 1.0) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/kll/tests.rs b/crates/accuracy/src/guarantee/kll/tests.rs new file mode 100644 index 000000000..02093b26f --- /dev/null +++ b/crates/accuracy/src/guarantee/kll/tests.rs @@ -0,0 +1,29 @@ +use super::*; + +/// A hand-built `k` above the library's cap gives no guarantee. +#[test] +fn a_k_above_the_library_cap_gives_no_guarantee() { + let g = |k| guarantee(&SketchParams::Kll { k }, KLL_CALIBRATED_FAILURE_PROBABILITY); + assert!(g(KLL_MAX_K).is_some()); + assert!(g(KLL_MAX_K + 1).is_none()); +} + +/// Below δ = 0.01 the error scales by the Gaussian quantile ratio +/// `z(1 − δ/2) / z(1 − 0.01/2)`. +#[test] +fn delta_scale_is_the_gaussian_quantile_ratio() { + assert_eq!(delta_scale(0.01), Some(1.0)); + assert_eq!(delta_scale(0.5), Some(1.0)); + let ratio = delta_scale(0.001).unwrap(); + assert!((ratio - 1.277_463_039_557_138_7).abs() < 1e-8, "{ratio}"); + assert!((normal_quantile(0.975) - 1.959_963_984_540_054).abs() < 1e-8); + // Far in the tail, `1 − δ/2` rounds to 1 (or near it) in f64, so the + // scale must be computed from the lower tail, `−z(δ/2)`. + for (delta, expected) in [ + (1e-16, 3.224_121_029_197_093_4), + (1e-15, 3.116_223_140_825_122_4), + ] { + let scale = delta_scale(delta).unwrap(); + assert!((scale - expected).abs() < 1e-8, "{delta}: {scale}"); + } +} diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 60a98efe7..0227d0008 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -1,67 +1,80 @@ //! Dispatch committed estimator parameters to their accuracy models. -use asap_types::accuracy::{ - AccuracyGuarantee, AccuracyMetric, BoundExpr, GuaranteeSource, ProbabilityExpr, -}; +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr}; use asap_types::ir::schema::FieldDataType; -use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::summary_type::{SketchParams, SketchStatistic}; -#[cfg(test)] -use crate::requirements::meets; use asap_types::summary_type::SummaryTypeCapability; -pub(crate) mod cardinality; -pub(crate) mod cms; -pub(crate) mod count_sketch; -pub(crate) mod ddsketch; -pub mod hll; -pub(crate) mod kll; -#[cfg(test)] -mod sizing_oracle; -pub mod univmon; +use crate::within_limit; + +mod binomial; +mod cardinality; +mod cms; +mod count_sketch; +mod ddsketch; +mod hll; +mod kll; +mod univmon; + +use binomial::binomial_upper_tail; -#[allow(dead_code)] // Read again for the error bound of each listed θ (`parameter_list`), in #1207 PRs 3–4. -pub(super) fn sketch_guarantee( - algorithm: &SketchAlgorithm, +/// The guarantee of θ for `query`: where one parameter sets both the error +/// and the failure probability (KLL's and KMV's `k`), the one that holds at +/// `failure_probability`; where structural parameters set them apart (a +/// Count-Min's width and depth, HLL's precision), the one θ fixes, whose +/// failure probability may exceed `failure_probability` or be unknown +/// (HLL's). A Hydra state reads each subpopulation's state, so it has +/// that state's guarantee. +fn guarantee_of_parameters( params: &SketchParams, query: &SketchStatistic, + failure_probability: f64, ) -> Option { match params { - SketchParams::Kll { .. } => kll::guarantee(algorithm, params, query), - SketchParams::DDSketch { .. } => ddsketch::guarantee(algorithm, params, query), - SketchParams::Hll { .. } => hll::generic_guarantee(algorithm, params, query), - SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => { - cms::guarantee(algorithm, params, query) + SketchParams::Kll { .. } => kll::guarantee(params, failure_probability), + SketchParams::Kmv { .. } => cardinality::guarantee(params, failure_probability), + SketchParams::Hydra(hydra) => { + guarantee_of_parameters(&hydra.per_subpopulation(), query, failure_probability) } + SketchParams::DDSketch { .. } => ddsketch::guarantee(params), + SketchParams::Hll { .. } => hll::guarantee(params), + SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => cms::guarantee(params), SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { - count_sketch::guarantee(algorithm, params, query) - } - SketchParams::Kmv { .. } => cardinality::guarantee(algorithm, params, query), - SketchParams::UnivMon { .. } => univmon::guarantee(algorithm, params, query), - SketchParams::Hydra(hydra) => { - sketch_guarantee(algorithm, &hydra.per_subpopulation(), query) + count_sketch::guarantee(params) } + SketchParams::UnivMon { .. } => univmon::guarantee(params, query), } } +/// The guarantee [`theoretical_guarantee`] gives, before it checks that θ +/// fails at most `failure_probability` rarely: its metric, and whether its +/// failure probability is known at all. `None` when the summary type does +/// not answer `statistic` or has no error model for it. +pub(crate) fn guarantee_before_failure_probability_check( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, +) -> Option { + if !matches!(summary_type, FieldDataType::Sketch(_)) + || !SummaryTypeCapability::of(summary_type).answers(statistic) + { + return None; + } + guarantee_of_parameters(parameters, statistic, failure_probability) +} + +/// The guarantee of an error at most `bound` in `metric`, failing with +/// probability at most `failure_probability`. fn bounded_guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, metric: AccuracyMetric, bound: f64, - delta: ProbabilityExpr, - contract: &str, + failure_probability: ProbabilityExpr, ) -> AccuracyGuarantee { AccuracyGuarantee::new( metric, BoundExpr::Constant { value: bound }, - delta, - vec![GuaranteeSource::SketchEvaluation { - algorithm: format!("{algorithm:?}"), - contract: contract.into(), - params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), - query: format!("{query:?}"), - }], + failure_probability, ) } @@ -69,7 +82,11 @@ fn bounded_guarantee( /// `summary_type` with parameters `parameters` at failure probability /// `failure_probability` (the guarantee's own failure probability): the /// algorithm's error formula (concepts doc, section 2, "Where the guarantee -/// comes from"). `None` when this θ cannot fail that rarely, when the +/// comes from"). Where one parameter sets both the error and the failure +/// probability (KLL's and KMV's `k`), the error is the one that holds at +/// `failure_probability`; where structural parameters set them apart (a +/// Count-Min's width and depth), the bound holds only when θ fails at most +/// that rarely. `None` when this θ cannot fail that rarely, when the /// summary type does not answer `statistic` (its capability row) or has no /// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch /// readouts without a certified bound). @@ -79,34 +96,27 @@ pub fn theoretical_guarantee( statistic: &SketchStatistic, failure_probability: f64, ) -> Option { - match summary_type { - FieldDataType::Sketch(_) if SummaryTypeCapability::of(summary_type).answers(statistic) => { - let _ = (parameters, failure_probability); - todo!("#1207 PR 4: the algorithm's error formula at θ, at failure_probability") - } - _ => None, - } + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + failure_probability, + ) + .filter(|guarantee| holds_at_failure_probability(guarantee, failure_probability)) } -/// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded -/// up by a relative 1e-12 so rounding never certifies a smaller tail. -pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { - if k == 0 || p >= 1.0 { - return 1.0; - } - if k > n || p <= 0.0 { - return 0.0; - } - let (ln_p, ln_q) = (p.ln(), (-p).ln_1p()); - let mut ln_choose = 0.0; - let mut tail = 0.0; - for i in 0..=n { - if i > 0 { - ln_choose += f64::from(n - i + 1).ln() - f64::from(i).ln(); - } - if i >= k { - tail += (ln_choose + f64::from(i) * ln_p + f64::from(n - i) * ln_q).exp(); - } - } - (tail * (1.0 + 1e-12)).min(1.0) +/// Whether `guarantee` fails at most `failure_probability` rarely: its own +/// failure probability is known and within it ([`within_limit`]). +pub(crate) fn holds_at_failure_probability( + guarantee: &AccuracyGuarantee, + failure_probability: f64, +) -> bool { + guarantee + .spec + .failure_probability + .evaluate() + .is_some_and(|own| within_limit(own, failure_probability)) } + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/sizing_oracle.rs b/crates/accuracy/src/guarantee/sizing_oracle.rs deleted file mode 100644 index 0df88e3cb..000000000 --- a/crates/accuracy/src/guarantee/sizing_oracle.rs +++ /dev/null @@ -1,676 +0,0 @@ -//! The closed-form sizing per summary type that the predefined parameter -//! lists (`asap_types::summary_type::parameter_list`) are checked against: -//! the parameters of a summary type for error `ε` at failure probability -//! `δ`. Test-only; the planner itself only reads the predefined lists. -use super::*; -use asap_types::ir::operator::AggIntent; -use asap_types::summary_type::KLL_MAX_K; - -/// The parameters of algorithm `kind` for error `eps` at failure -/// probability `delta`. The predefined lists (`parameter_list`) are this -/// sizing over an error grid, which a test keeps equal. -pub(super) fn size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - match kind { - SketchAlgorithm::UnivMon => univmon_params(intent, eps, delta), - SketchAlgorithm::Kll => kll_params(eps, delta), - SketchAlgorithm::Cms => SketchParams::Cms { - width: cms_width(eps), - depth: cms_depth(delta), - }, - SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll_precision(eps, delta), - }, - SketchAlgorithm::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), - heap_size: topk_capacity(k, eps), - } - } - // Non-preferred candidates (DDSketch / Kmv / CountSketch / - // CountSketchWithHeap) are only reachable once a cost model picks - // them; sized here so that wiring is local. - SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: eps }, - SketchAlgorithm::Kmv => SketchParams::Kmv { - k: kmv_k(eps, delta), - }, - SketchAlgorithm::CountSketch => { - let width = count_sketch_width(eps); - SketchParams::CountSketch { - width, - depth: count_sketch_depth(width, delta), - } - } - SketchAlgorithm::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - let width = count_sketch_width(eps); - SketchParams::CountSketchWithHeap { - width, - depth: count_sketch_depth(width, delta), - heap_size: topk_capacity(k, eps), - } - } - // Listed, not `_`: a family `asap_sketchlib` adds fails to compile - // here until the planner sizes it or lists it as not offered. - SketchAlgorithm::CountSketchWithL2Norm - | SketchAlgorithm::CmsWithSpaceSaving - | SketchAlgorithm::CountSketchWithSpaceSaving - | SketchAlgorithm::CmsFold - | SketchAlgorithm::CountSketchFold - | SketchAlgorithm::HllHip - | SketchAlgorithm::KllDynamic - | SketchAlgorithm::SpaceSaving - | SketchAlgorithm::Elastic - | SketchAlgorithm::Coco - | SketchAlgorithm::Bloom - | SketchAlgorithm::SetAggregator - | SketchAlgorithm::UnivMonTwoTier - | SketchAlgorithm::UnivMonQ - | SketchAlgorithm::HydraCms - | SketchAlgorithm::HydraCountSketch - | SketchAlgorithm::HydraKll - | SketchAlgorithm::HydraHll - | SketchAlgorithm::HydraUnivMon => { - unreachable!("Pass 1 offers only the families in ALGORITHM_PREFERENCE, not {kind:?}") - } - } -} - -/// Accuracy-dependent candidate budget, not a completeness theorem. The -/// membership model must still certify the selected set independently. -fn topk_capacity(k: usize, eps: f64) -> u32 { - u32::try_from(k) - .unwrap_or(u32::MAX) - .max(saturating_ceil(1.0 / eps, 1, 1 << 26)) -} - -/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` -/// (a degenerate ε means "as accurate as this family goes"). -fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { - if !x.is_finite() || x <= 0.0 { - return hi; - } - (x.ceil() as u32).clamp(lo, hi) -} - -// Count-Min Sketch. - -/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. -fn cms_width(eps: f64) -> u32 { - saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) -} - -/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. -/// δ = 0.01 → depth 5. -fn cms_depth(delta: f64) -> u32 { - saturating_ceil((1.0 / delta).ln(), 1, 32) -} - -// Count Sketch. - -/// CountSketch `L2` point-query width: ε = sqrt(3/w). -pub(super) fn count_sketch_width(eps: f64) -> u32 { - saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) -} - -/// The smallest positive odd depth whose binomial median failure -/// probability at `width` is at most `delta`, capped at 255. -pub(super) fn count_sketch_depth(width: u32, delta: f64) -> u32 { - if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { - return 255; - } - (1..=255) - .step_by(2) - .find(|&depth| { - count_sketch::count_sketch_failure_probability(width, depth) - .is_some_and(|tail| tail <= delta) - }) - .unwrap_or(255) -} - -// KMV. - -/// The largest `k` sizing returns. -const KMV_MAX_K: u32 = 1 << 26; - -/// The smallest `k` whose KMV failure probability at relative error `eps` -/// is at most `delta` ([`cardinality::kmv_failure_probability`]), up to -/// `2^26`; a degenerate `eps` or `delta` saturates to `2^26`. -fn kmv_k(eps: f64, delta: f64) -> u32 { - use cardinality::{kmv_failure_probability, KMV_MIN_K}; - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return KMV_MAX_K; - } - // The overestimate term alone at most `δ` bounds `k` below; both terms - // at most `δ/2` bound it above. - let k_at = |ln_inverse: f64| { - saturating_ceil( - 1.0 + (1.0 + eps) * (2.0 + eps) * ln_inverse / (eps * eps), - KMV_MIN_K, - KMV_MAX_K, - ) - }; - let (mut lo, mut hi) = (k_at((1.0 / delta).ln()), k_at((2.0 / delta).ln())); - if kmv_failure_probability(hi, eps) > delta { - return KMV_MAX_K; - } - while lo < hi { - let mid = lo + (hi - lo) / 2; - if kmv_failure_probability(mid, eps) <= delta { - hi = mid; - } else { - lo = mid + 1; - } - } - hi -} - -// HLL. - -/// Classic HLL's precision for relative error `eps` with failure -/// probability `delta`, without data. Below `1/ε` distinct items a single -/// bucket collision already exceeds the error, and the chance of one is -/// about `N²/(2m)`; a certificate for any bound of at least `1/ε` -/// therefore needs `m ≥ 1/(2ε²δ)`. This is the smallest such precision at -/// which [`hll::ClassicHllConfidence`] certifies `delta` at `eps` for every -/// bound up to `⌈1/ε⌉` (18 when none up to 18 does). -fn hll_precision(eps: f64, delta: f64) -> u8 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return 18; - } - let start = saturating_ceil((1.0 / (2.0 * eps * eps * delta.min(1.0))).log2(), 4, 18) as u8; - let reference = (1.0 / eps) - .ceil() - .min(f64::from(hll::MAX_CONTRACT_DISTINCT)) as u64; - (start..=18) - .find(|&p| hll::classic_failure_probability(p, eps, reference) <= delta) - .unwrap_or(18) -} - -/// The smallest precision, at least `at_least`, whose classic failure -/// probability at `relative_error` over a source of at most `max_distinct` -/// distinct items is at most `failure_probability`; `at_least` when none up -/// to 18 is: [`hll_precision`]'s rule for a source's own distinct bound. -fn min_precision( - relative_error: f64, - failure_probability: f64, - max_distinct: u64, - at_least: u8, -) -> u8 { - (at_least..=18) - .find(|&p| { - hll::classic_failure_probability(p, relative_error, max_distinct) <= failure_probability - }) - .unwrap_or(at_least) -} - -/// The smallest precision at which classic HLL certifies failure -/// probability `delta` at `relative_error` over at most `max_distinct` -/// distinct items ([`hll::ClassicHllConfidence`]); `None` when `delta` is -/// not in (0, 1), the bound or error is outside the contract, or no -/// precision up to 18 certifies it. -fn classic_hll_precision(max_distinct: u32, relative_error: f64, delta: f64) -> Option { - if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { - return None; - } - let model = hll::ClassicHllConfidence::new(max_distinct, relative_error)?; - (4..=hll::MAX_PRECISION).find(|&p| model.failure_probability(p).is_some_and(|d| d <= delta)) -} - -// KLL. - -/// Invert Apache DataSketches' empirical 99th-percentile, single-sided KLL -/// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. Capped at -/// [`KLL_MAX_K`], the largest `k` `asap_sketchlib` builds; an `eps` that -/// needs more gets that `k`, whose guarantee then misses the target. -pub(super) fn kll_k(eps: f64) -> u32 { - saturating_ceil( - (kll::KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / kll::KLL_RANK_ERROR_EXPONENT_99), - 8, - KLL_MAX_K, - ) -} - -/// `k` for rank error `eps` at failure probability `delta`: the empirical -/// 99% calibration for `delta >= 0.01`; below, the calibration at -/// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. -pub(super) fn kll_params(eps: f64, delta: f64) -> SketchParams { - if delta.is_nan() || delta <= 0.0 || delta >= kll::KLL_CALIBRATED_FAILURE_PROBABILITY { - return SketchParams::Kll { k: kll_k(eps) }; - } - SketchParams::Kll { - k: kll_k(eps / kll::delta_scale(delta).unwrap_or(1.0)), - } -} - -// UnivMon. - -/// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and -/// seed 19 picks the layers a key reaches. -const UNIVMON_MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; -/// The most rows an L2 sketch is sized with: the deepest odd shape within -/// the 20 rows the executor builds ([`univmon::hash_independent`]), so the -/// row median is one row's estimate, and within the hash budget at the -/// more than 20 columns an L2 guarantee needs (5 column bits: -/// 19 · 6 = 114 ≤ 128). -const UNIVMON_MAX_L2_ROWS: u32 = 19; -/// Each row of a layer reads its column and sign bits from one hash. -const UNIVMON_HASH_BITS: u32 = 128; -/// The largest distinct-count heap sizing returns. -const UNIVMON_MAX_HEAP: u32 = 1 << 26; - -/// The shape baseline for readouts without a size model: the total (exact -/// whatever the shape) and entropy (uncertified). -pub(super) const UNIVMON_BASELINE: SketchParams = SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1024, - layers: 16, -}; - -/// The smallest rows and columns whose L2 guarantee meets `(eps, delta)`, -/// clamped to the hash budget: with more rows than -/// [`UNIVMON_MAX_L2_ROWS`], or more columns than `rows` leave bits for, the -/// shape is the largest one built and its guarantee misses the target. -fn univmon_l2_shape(eps: f64, delta: f64) -> (u32, u32) { - use univmon::{l2_failure_probability, l2_relative_error, L2_ROW_FAILURE}; - let rows = (1..=UNIVMON_MAX_L2_ROWS) - .find(|&rows| l2_failure_probability(rows) <= delta) - .unwrap_or(UNIVMON_MAX_L2_ROWS); - let max_bits = (UNIVMON_HASH_BITS / rows - 1).min(26); - let max_cols = 1u32 << max_bits; - let meets = |cols: u32| l2_relative_error(cols).is_some_and(|error| error <= eps); - if !(eps.is_finite() && eps > 0.0) || !meets(max_cols) { - return (rows, max_cols); - } - // The norm's error `ε` needs `ε₂ ≤ 1 − (1 − ε)²`; a power-of-two width - // needs `2/(0.1 ε₂²)` columns and any other `9/(4 · 0.1 ε₂²)`. - let f2_error = 1.0 - (1.0 - eps.min(1.0)).powi(2); - let general = 9.0 / (4.0 * L2_ROW_FAILURE * f2_error * f2_error); - // Off powers of two the error falls as the width grows, and a power of - // two meets the target whenever its non-power-of-two neighbours do, so - // walking down from the estimate stops at the smallest such width. - let mut general = saturating_ceil(general, 1, max_cols); - while general > 1 && meets(general - 1) { - general -= 1; - } - while !meets(general) { - general += 1; - } - let power_of_two = (0..=max_bits) - .map(|bits| 1u32 << bits) - .find(|&cols| meets(cols)) - .unwrap_or(max_cols); - (rows, general.min(power_of_two)) -} - -/// The Chebyshev share of the distinct count's failure probability at -/// heap `k` and relative error `eps`: `1.35/(k ε²)`. -fn univmon_distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { - univmon::DISTINCT_VARIANCE / (f64::from(heap_size) * eps * eps) -} - -/// The smallest heap `k ≥ 8` whose distinct count misses by more than -/// `eps · F0` with probability at most `delta`, top-layer completeness -/// included: `1.35/(k ε²) + (e/4)^(k/2) ≤ δ`. Capped; a capped heap's -/// guarantee misses the target. -fn univmon_distinct_heap(eps: f64, delta: f64) -> u32 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return UNIVMON_MAX_HEAP; - } - let meets = |k: u32| { - univmon_distinct_chebyshev(k, eps) + univmon::incomplete_top_layer_bound(k) <= delta - }; - let mut k = saturating_ceil( - univmon::DISTINCT_VARIANCE / (delta * eps * eps), - univmon::DISTINCT_MIN_HEAP, - UNIVMON_MAX_HEAP, - ); - while !meets(k) && k < UNIVMON_MAX_HEAP { - k += 1; - } - k -} - -/// Size a UnivMon for one readout's `(eps, delta)`: -/// - L2: the smallest rows and columns meeting the target -/// ([`univmon_l2_shape`]), one layer (L2 reads only layer 0) and a -/// one-key heap; -/// - distinct count: the smallest heap meeting it -/// ([`univmon_distinct_heap`]) and the most layers, since the layers the -/// top layer needs depend on the source's distinct bound, which plan -/// selection binds. The bound holds whatever the Count Sketch keeps in -/// the heaps, so each layer's is one counter; -/// - total and entropy: the baseline shape (the total is exact; entropy -/// has no certified bound). -fn univmon_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { - match intent { - AggIntent::FrequencyL2 { .. } => { - let (sketch_rows, sketch_cols) = univmon_l2_shape(eps, delta); - SketchParams::UnivMon { - heap_size: 1, - sketch_rows, - sketch_cols, - layers: 1, - } - } - AggIntent::Cardinality { .. } => SketchParams::UnivMon { - heap_size: univmon_distinct_heap(eps, delta), - sketch_rows: 1, - sketch_cols: 1, - layers: UNIVMON_MAX_LAYERS, - }, - _ => UNIVMON_BASELINE, - } -} - -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; - - /// Classic HLL's precision grows until its failure probability over the - /// source's distinct bound meets the target, up to 18. - #[test] - fn classic_hll_precision_grows_for_its_failure_probability() { - let p = min_precision(0.05, 0.01, 64, 4); - assert!(p > 4); - assert!(hll::classic_failure_probability(p, 0.05, 64) <= 0.01); - assert!(hll::classic_failure_probability(p - 1, 0.05, 64) > 0.01); - assert_eq!(min_precision(0.05, 1e-30, 4096, 12), 12); - } - - /// A supported estimator contract supplies a probability, unlike - /// generic HLL RSE: sized for δ = 0.01 over at most 128 distinct items, - /// the classic guarantee binds to a known failure probability. - #[test] - fn bounded_classic_hll_has_a_feasible_confidence_target() { - let precision = - classic_hll_precision(128, 0.05, 0.01).expect("finite confidence-sized HLL"); - let guarantee = hll::classic_guarantee(precision, Some(0.05), "m") - .unwrap() - .bind(&|statistic| { - let (precision, eps, _) = hll::parse_classic_failure_statistic(statistic)?; - Some(hll::classic_failure_probability(precision, eps, 128)) - }); - assert!(!guarantee.has_unknown()); - assert!(guarantee.spec.failure_probability.evaluate().unwrap() <= 0.01); - assert_eq!(guarantee.spec.error.evaluate(), Some(0.05)); - } - - /// Tighter confidence must increase precision or explicitly become unavailable. - #[test] - fn sizing_and_domain_limits_are_consistent() { - let model = hll::ClassicHllConfidence::new(128, 0.05).unwrap(); - let precision = |delta| classic_hll_precision(128, 0.05, delta); - assert!(precision(0.001).unwrap() > precision(0.01).unwrap()); - assert!(precision(1e-12).is_none()); - assert!(precision(0.0).is_none()); - assert!(precision(f64::NAN).is_none()); - assert!(model.failure_probability(3).is_none()); - assert!(model.failure_probability(19).is_none()); - assert!(model.failure_probability(7).is_none()); - for (n, e) in [ - (0, 0.05), - (hll::MAX_CONTRACT_DISTINCT + 1, 0.05), - (128, 0.0), - (128, f64::NAN), - ] { - assert!(hll::ClassicHllConfidence::new(n, e).is_none()); - } - } - - /// Each algorithm's predefined list (`parameter_list`) is this sizing - /// over the error grid ε = 0.1 … 0.001 at failure probability 0.01, - /// smallest first, without repeats; KMV's and UnivMon's distinct counts - /// stop at ε = 0.01. Count-Min lists each sized width with depths 5, 7, - /// 10 and 12, Count Sketch with 47, 81, 117 and 155, and a heap variant - /// each of those with - /// every heap size of the 1-2-5 series 10 … 10 000. UnivMon's list also - /// combines each L2 shape with each distinct-count shape (28 entries). - #[test] - fn parameter_list_is_the_sizing_over_the_error_grid() { - use asap_types::summary_type::parameter_list; - let full_grid = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; - let distinct_grid = [0.1, 0.05, 0.02, 0.01]; - let cms_depths = [5, 7, 10, 12]; - let count_sketch_depths = [47, 81, 117, 155]; - let heap_sizes = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; - let l2 = AggIntent::FrequencyL2 { col: None }; - let top_1 = AggIntent::TopK { k: 1 }; - let quantile = default_quantile(0.5); - let distinct = default_cardinality(); - for (algorithm, intents) in [ - (SketchAlgorithm::Kll, vec![&quantile]), - (SketchAlgorithm::DDSketch, vec![&quantile]), - (SketchAlgorithm::Hll, vec![&distinct]), - (SketchAlgorithm::Kmv, vec![&distinct]), - (SketchAlgorithm::Cms, vec![&distinct]), - (SketchAlgorithm::CountSketch, vec![&distinct]), - (SketchAlgorithm::CmsWithHeap, vec![&top_1]), - (SketchAlgorithm::CountSketchWithHeap, vec![&top_1]), - (SketchAlgorithm::UnivMon, vec![&l2, &distinct, &quantile]), - ] { - let mut sized: Vec = Vec::new(); - for intent in intents { - let grid: &[f64] = match (algorithm, intent) { - ( - SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon, - AggIntent::Cardinality { .. }, - ) => &distinct_grid, - _ => &full_grid, - }; - for &eps in grid { - let listed: Vec = match size_params(algorithm, intent, eps, 0.01) - { - SketchParams::Cms { width, .. } => cms_depths - .iter() - .map(|&depth| SketchParams::Cms { width, depth }) - .collect(), - SketchParams::CountSketch { width, .. } => count_sketch_depths - .iter() - .map(|&depth| SketchParams::CountSketch { width, depth }) - .collect(), - SketchParams::CmsWithHeap { width, .. } => cms_depths - .iter() - .flat_map(|&depth| { - heap_sizes - .iter() - .map(move |&heap_size| SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }) - }) - .collect(), - SketchParams::CountSketchWithHeap { width, .. } => count_sketch_depths - .iter() - .flat_map(|&depth| { - heap_sizes.iter().map(move |&heap_size| { - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } - }) - }) - .collect(), - other => vec![other], - }; - for params in listed { - if !sized.contains(¶ms) { - sized.push(params); - } - } - } - } - let list = parameter_list(algorithm); - let (listed_sized, combined): (Vec<_>, Vec<_>) = - list.iter().partition(|params| sized.contains(params)); - assert_eq!( - format!("{listed_sized:?}"), - format!("{:?}", sized.iter().collect::>()), - "{algorithm:?}" - ); - let expected_combined = if algorithm == SketchAlgorithm::UnivMon { - 28 - } else { - 0 - }; - assert_eq!(combined.len(), expected_combined, "{algorithm:?}"); - } - } - - /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. - #[test] - fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count; - assert_eq!( - size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), - SketchParams::Cms { - width: 2719, - depth: 7 - } // ⌈e/0.001⌉, ⌈ln 1000⌉ - ); - assert_eq!( - size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), - SketchParams::Cms { - width: 2719, - depth: 5 - } - ); - assert_eq!( - size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), - SketchParams::Cms { - width: 272, - depth: 5 - } - ); - } - - /// A top-k heap holds `max(k, ⌈1/ε⌉)` entries over a CMS sized as for counts. - #[test] - fn topk_heap_capacity_respects_accuracy_and_output_count() { - let intent = AggIntent::TopK { k: 25 }; - assert_eq!( - size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.01, 0.01), - SketchParams::CmsWithHeap { - width: 272, // ⌈e/0.01⌉ - depth: 5, - heap_size: 100, - } - ); - } - - /// The executor's `CountSketch<_, RegularPath>` takes a column as the - /// low 32 hash bits modulo the width, so a row fails with probability - /// `1/3` (to within `2^-32`) at any width, not `3/8` off powers of two: - /// ε = 0.05 (width 1200) needs depth 47 at δ = 0.01, 81 at 1e-3 and - /// 117 at 1e-4, not 85, 149 and 215. - #[test] - fn depth_follows_the_libraries_modulo_column_hash() { - let width = count_sketch_width(0.05); - assert_eq!(width, 1200); - assert!((count_sketch::row_failure_probability(width) - 1.0 / 3.0).abs() < 1e-6); - assert_eq!( - [0.01, 1e-3, 1e-4].map(|delta| count_sketch_depth(width, delta)), - [47, 81, 117] - ); - } - - /// The depth is the smallest odd one whose binomial tail meets δ. Each - /// is at most Hoeffding's `ln(1/δ) / (2 (1/2 − p)²)` at the row's - /// failure probability `p ≈ 1/3`: `18 ln(1/δ)` at any width. - #[test] - fn count_sketch_depth_is_the_smallest_meeting_the_binomial_tail() { - use count_sketch::count_sketch_failure_probability; - for width in [1024, 3000, 30_000] { - let hoeffding = 18.0 * (1.0 + 1e-4); - for delta in [0.1, 0.01, 0.001] { - let depth = count_sketch_depth(width, delta); - assert!(depth % 2 == 1); - assert!(count_sketch_failure_probability(width, depth).unwrap() <= delta); - if depth > 1 { - assert!(count_sketch_failure_probability(width, depth - 2).unwrap() > delta); - } - assert!( - f64::from(depth) <= hoeffding * (1.0 / delta).ln(), - "{width} {delta}" - ); - } - } - // Any width fails a row with probability 1/3 (to within 2^-32 w/3): - // P(Bin(3, 1/3) ≥ 2) = 7/27. - for width in [1024, 1000] { - let tail = count_sketch_failure_probability(width, 3).unwrap(); - assert!((tail - 7.0 / 27.0).abs() < 1e-6, "{width}"); - } - assert_eq!(count_sketch_failure_probability(1000, 4), None); - } - - /// Pass 1 sizes classic HLL from (ε, δ) without data: the smallest - /// precision with `m ≥ 1/(2ε²δ)` that the tail formula certifies at the - /// target ε, which its guarantee states as its error bound. - #[test] - fn precision_is_sized_from_epsilon_and_delta() { - for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { - let params = size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); - let SketchParams::Hll { precision: sized } = params else { - panic!("{params:?}") - }; - assert_eq!(sized, precision); - } - } - - /// `k` never exceeds the 26 602 `asap_sketchlib` silently caps it at, so - /// a guarantee never claims a larger `k` than runs. - #[test] - fn kll_k_is_capped_at_the_library_maximum() { - assert_eq!(kll_k(1e-6), KLL_MAX_K); - assert_eq!(KLL_MAX_K, 26_602); - } - - /// KLL `k` inverts the rank-error fit: ε = 0.01 → 269, ε = 0.05 → 52. - #[test] - fn accuracy_target_drives_kll_k() { - let q = default_quantile(0.99); - assert_eq!( - size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01), - SketchParams::Kll { k: 269 } - ); - assert_eq!( - size_params(SketchAlgorithm::Kll, &q, 0.05, 0.01), - SketchParams::Kll { k: 52 } - ); - } - - /// A zero ε saturates KLL `k` to the tightest buildable value. - #[test] - fn degenerate_epsilon_saturates_to_tightest_kll_params() { - assert_eq!( - size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.0, 0.01), - SketchParams::Kll { k: 26_602 } - ); - } - - /// Example 2's Q1 (ε = 0.02, δ = 0.01) needs a heap of 337,500 keys, - /// `⌈1.35/(δ ε²)⌉`: the top layer's term is negligible at that heap. - /// At (0.5, 0.5) it is not: 1.35/(12 · 0.25) + (e/4)^6 > 0.5, while 13 - /// keys give 0.415 + 0.081. - #[test] - fn distinct_heap_for_example2_q1() { - assert_eq!(univmon_distinct_heap(0.02, 0.01), 337_500); - assert_eq!(univmon_distinct_heap(0.5, 0.5), 13); - } -} diff --git a/crates/accuracy/src/guarantee/tests.rs b/crates/accuracy/src/guarantee/tests.rs new file mode 100644 index 000000000..f53608525 --- /dev/null +++ b/crates/accuracy/src/guarantee/tests.rs @@ -0,0 +1,100 @@ +use super::*; +use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::summary_type::SketchAlgorithm; + +/// Every listed θ has a theoretical guarantee for the statistic its +/// algorithm is listed for, at failure probability 0.01, in the +/// algorithm's metric; along a list of one structure the error never +/// grows (the lists run weakest first). +#[test] +fn every_listed_parameters_has_a_guarantee_at_one_percent() { + use asap_types::summary_type::parameter_list; + let point = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: Some("1".into()), + }; + for (algorithm, statistic, metric) in [ + ( + SketchAlgorithm::Kll, + SketchStatistic::Quantile { q: 0.5 }, + AccuracyMetric::Rank, + ), + ( + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.5 }, + AccuracyMetric::Relative, + ), + ( + SketchAlgorithm::Kmv, + SketchStatistic::Cardinality, + AccuracyMetric::Relative, + ), + ( + SketchAlgorithm::Cms, + point.clone(), + AccuracyMetric::ABSOLUTE_STREAM_L1, + ), + ( + SketchAlgorithm::CountSketch, + point, + AccuracyMetric::ABSOLUTE_STREAM_L2, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 10 }, + AccuracyMetric::ABSOLUTE_STREAM_L1, + ), + ] { + let summary_type = FieldDataType::Sketch(algorithm); + let errors: Vec = parameter_list(algorithm) + .iter() + .map(|parameters| { + let guarantee = theoretical_guarantee(&summary_type, parameters, &statistic, 0.01) + .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); + assert_eq!(guarantee.spec.accuracy_metric, metric, "{algorithm:?}"); + guarantee.spec.error.evaluate().unwrap() + }) + .collect(); + assert!(!errors.is_empty(), "{algorithm:?}"); + if matches!( + algorithm, + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch | SketchAlgorithm::Kmv + ) { + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{algorithm:?}: {errors:?}" + ); + } + } +} + +/// One parameter sets both (KLL's k): its rank error grows as the +/// required failure probability shrinks. Separate parameters (CMS width +/// and depth): the bound holds only when the depth fails rarely enough. +#[test] +fn a_guarantee_is_evaluated_at_the_required_failure_probability() { + let quantile = SketchStatistic::Quantile { q: 0.5 }; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let k = SketchParams::Kll { k: 547 }; + let error_at = |delta| { + theoretical_guarantee(&kll, &k, &quantile, delta) + .unwrap() + .spec + .error + .evaluate() + .unwrap() + }; + assert_eq!(error_at(1.0), error_at(0.01)); + assert!(error_at(1e-6) > error_at(0.01)); + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms); + let point = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let depth = |depth| SketchParams::Cms { width: 272, depth }; + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4).is_none()); + assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4).is_some()); + // A statistic the summary type does not answer has no guarantee. + assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01).is_none()); +} diff --git a/crates/accuracy/src/guarantee/univmon.rs b/crates/accuracy/src/guarantee/univmon.rs deleted file mode 100644 index 55ec1bf58..000000000 --- a/crates/accuracy/src/guarantee/univmon.rs +++ /dev/null @@ -1,480 +0,0 @@ -//! UnivMon's readouts and the guarantees `asap_sketchlib`'s standard-mode -//! estimators carry (the derivations are in the library's -//! `docs/api/api_univmon.md`, the bounds `estimate_l2` and -//! `estimate_cardinality` cite): -//! - the total (a bare `PointCount`) is the exact unit-update count; -//! - L2 reads layer 0's Count Sketch directly: per row, Chebyshev with -//! `Var ≤ 2q F2²`, and the median of the rows fails only when at least -//! `⌈r/2⌉` rows fail, an exact binomial tail; -//! - the distinct count is unbiased with `Var ≤ 1.35 F0²/k` for a heap of -//! `k ≥ 8` once the top layer is complete, which needs the source's -//! distinct bound, so its guarantee is built where the source is known -//! (`distinct_guarantee`) and plan selection binds it; -//! - entropy has no certified bound. -//! -//! Every bound assumes the key hash is uniformly random and the shape keeps -//! the hashes independent (`hash_independent`). Terminal-mode states -//! (OctoSketch aggregators fed promoted deltas) are not covered: the -//! executor rejects them. -use super::*; - -/// The per-row failure probability the L2 guarantee fixes: a row misses by -/// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it -/// lets a shape state one `(ε, δ)`: the columns give `ε`, the rows `δ`. -pub(super) const L2_ROW_FAILURE: f64 = 0.1; -/// `Var[estimate_cardinality] ≤ 1.35 F0²/k` for `k ≥ 8`, the constant the library -/// proves (it also states the looser `2 F0²/k`). -pub(super) const DISTINCT_VARIANCE: f64 = 1.35; -/// The smallest heap the distinct-count variance bound holds for. -pub(super) const DISTINCT_MIN_HEAP: u32 = 8; - -const L2_CONTRACT: &str = "univmon_layer0_f2_median_chebyshev_binomial_v1"; -const DISTINCT_CONTRACT: &str = "univmon_distinct_complete_top_layer_chebyshev_v1"; -/// Names the probability that the top layer is incomplete, left symbolic -/// in `max_distinct[source]` ([`incomplete_top_layer_statistic`]). -const INCOMPLETE_TOP_LAYER: &str = "univmon_incomplete_top_layer_probability"; -const MAX_DISTINCT: &str = "max_distinct"; - -pub(super) fn guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { - let SketchParams::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - return None; - }; - match query { - SketchStatistic::PointCount { value: None, .. } => { - Some(AccuracyGuarantee::exact("univmon_unit_update_total")) - } - SketchStatistic::FrequencyL2 if hash_independent(*sketch_rows, *sketch_cols, *layers) => { - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::Relative, - l2_relative_error(*sketch_cols)?, - ProbabilityExpr::Constant { - value: l2_failure_probability(*sketch_rows), - }, - L2_CONTRACT, - )) - } - // The distinct count needs the source's distinct bound - // ([`distinct_guarantee`]); entropy has no certified bound. - _ => None, - } -} - -/// Whether the executor builds this shape, with independent hashes: at -/// most `UNIVMON_MAX_LAYERS` (the library's) layers and `MATRIX_MAX_ROWS` (20) rows, the rows a -/// UnivMon state encodes, and `rows · (⌈log2 cols⌉ + 1) ≤ 128` -/// ([`asap_sketchlib::UnivMon::check_shape`]; the heap size is not part of -/// the hash rules). -pub(crate) fn hash_independent(rows: u32, cols: u32, layers: u8) -> bool { - rows as usize <= asap_sketchlib::MATRIX_MAX_ROWS - && asap_sketchlib::UnivMon::check_shape( - 1, - rows as usize, - cols as usize, - usize::from(layers), - ) - .is_ok() -} - -/// The relative error of the L2 norm at `cols` columns. Per row, the -/// squared norm misses by more than `ε₂ F2` with probability at most -/// `2q/ε₂²` ([`column_collision`]), so fixing that at -/// [`L2_ROW_FAILURE`] gives `ε₂ = √(2q/0.1)`. The norm then misses by at -/// most `1 − √(1 − ε₂)` (the low side; the high side `√(1 + ε₂) − 1` is -/// smaller). `None` when `ε₂ ≥ 1`. -pub(super) fn l2_relative_error(cols: u32) -> Option { - let f2_error = (2.0 * column_collision(cols) / L2_ROW_FAILURE).sqrt(); - (f2_error < 1.0).then(|| 1.0 - (1.0 - f2_error).sqrt()) -} - -/// `P(c_i = c_j)` for two keys in a row of a UnivMon layer of `cols` -/// columns. `asap_sketchlib` takes a UnivMon column as a `⌈log2 cols⌉`-bit -/// slice of one 128-bit hash modulo `cols` ([`hash_independent`]): uniform -/// for a power of two, and otherwise at most `9/(8 cols)` (the slice values -/// past `cols` fold onto the low columns; the worst case, -/// `cols = ¾ · 2^⌈log2 cols⌉`, doubles a third of them). A standalone -/// Count Sketch hashes modulo `cols` instead (`count_sketch`). -fn column_collision(cols: u32) -> f64 { - let cols = cols.max(1); - if cols.is_power_of_two() { - 1.0 / f64::from(cols) - } else { - 9.0 / (8.0 * f64::from(cols)) - } -} - -/// The median of `rows` independent rows misses only when at least -/// `⌈rows/2⌉` of them miss: `P(Binomial(rows, 0.1) ≥ ⌈rows/2⌉)`. -pub(super) fn l2_failure_probability(rows: u32) -> f64 { - super::binomial_upper_tail(rows, L2_ROW_FAILURE, rows.div_ceil(2)) -} - -/// `(e/4)^(k/2)`: the chance the top layer of a heap-`k` pyramid misses a -/// key, once the expected keys reaching it are at most `k/2` (Chernoff). -pub(super) fn incomplete_top_layer_bound(heap_size: u32) -> f64 { - (f64::from(heap_size) / 2.0 * (1.0 - 2.0 * std::f64::consts::LN_2)).exp() -} - -/// The statistic a UnivMon distinct count over `source` leaves its -/// top-layer failure probability as. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: &str) -> String { - format!("{INCOMPLETE_TOP_LAYER}[k={heap_size},layers={layers}]({MAX_DISTINCT}[{source}])") -} - -/// The heap, layers and source an `incomplete_top_layer_statistic` names. -pub fn parse_incomplete_top_layer_statistic(statistic: &str) -> Option<(u32, u8, &str)> { - let rest = statistic - .strip_prefix(INCOMPLETE_TOP_LAYER)? - .strip_prefix("[k=")?; - let (heap_size, rest) = rest.split_once(",layers=")?; - let (layers, rest) = rest.split_once(']')?; - let source = rest - .strip_prefix('(')? - .strip_prefix(MAX_DISTINCT)? - .strip_prefix('[')? - .strip_suffix("])")?; - Some((heap_size.parse().ok()?, layers.parse().ok()?, source)) -} - -/// The probability an `incomplete_top_layer_statistic` names, given the -/// source's distinct bound `F0_max`: -/// - 0 when `F0_max ≤ k`: the top layer cannot receive more keys than its -/// heap holds; -/// - `(e/4)^(k/2)` when `layers ≥ ⌈log2(F0_max/k)⌉ + 2`, i.e. -/// `F0_max ≤ k · 2^(layers−2)`, so at most `k/2` keys reach the top layer -/// in expectation; -/// - 1 (no certificate) otherwise: the pyramid is too shallow. -pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct: u64) -> f64 { - let (k, max_distinct) = (u128::from(heap_size), u128::from(max_distinct)); - if max_distinct <= k { - 0.0 - } else if layers >= 2 && max_distinct <= k << (layers - 2) { - incomplete_top_layer_bound(heap_size) - } else { - 1.0 - } -} - -/// A UnivMon distinct count over `source`'s own rows, at failure -/// probability `delta`: relative error at most `√(1.35/(k δ_c))`, where -/// `δ_c` is the share of `delta` left after the top layer's, failing with -/// probability `δ_c` plus the chance of an incomplete top layer, left -/// symbolic in the source's distinct bound. `None` for a heap under 8, a -/// `delta` too small for the heap, or a shape the library would not build. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn distinct_guarantee( - params: &SketchParams, - delta: f64, - source: &str, -) -> Option { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } = params - else { - return None; - }; - if *heap_size < DISTINCT_MIN_HEAP - || !hash_independent(*sketch_rows, *sketch_cols, *layers) - || !(delta > 0.0 && delta < 1.0) - { - return None; - } - let chebyshev = delta - incomplete_top_layer_bound(*heap_size); - if chebyshev <= 0.0 { - return None; - } - let relative_error = (DISTINCT_VARIANCE / (f64::from(*heap_size) * chebyshev)).sqrt(); - let statistic = incomplete_top_layer_statistic(*heap_size, *layers, source); - Some(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: relative_error, - }, - ProbabilityExpr::UnionBound { - terms: vec![ - ProbabilityExpr::Constant { value: chebyshev }, - ProbabilityExpr::Unknown { statistic }, - ], - }, - vec![GuaranteeSource::SketchEvaluation { - algorithm: "UnivMon".into(), - contract: DISTINCT_CONTRACT.into(), - params: serde_json::json!({"heap_size": heap_size, "layers": layers, - "max_distinct": format!("{MAX_DISTINCT}[{source}]"), - "relative_error": relative_error, - "variance": "1.35 F0^2 / k", - "hash_assumption": "independent_uniform_layer_hashes", - "mode": "standard", - "population_scope": "complete_evaluation_including_merged_tumbling_windows"}), - query: "Cardinality".into(), - }], - )) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::guarantee::sizing_oracle::{size_params, UNIVMON_BASELINE}; - use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::operator::AggIntent; - - fn l2_intent() -> AggIntent { - AggIntent::FrequencyL2 { col: None } - } - - fn univmon(heap_size: u32, rows: u32, cols: u32, layers: u8) -> SketchParams { - SketchParams::UnivMon { - heap_size, - sketch_rows: rows, - sketch_cols: cols, - layers, - } - } - - fn l2(params: &SketchParams) -> Option { - guarantee( - &SketchAlgorithm::UnivMon, - params, - &SketchStatistic::FrequencyL2, - ) - } - - fn with(params: &SketchParams, f: impl FnOnce(&mut u32, &mut u32, &mut u32)) -> SketchParams { - let mut params = params.clone(); - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - .. - } = &mut params - else { - unreachable!() - }; - f(heap_size, sketch_rows, sketch_cols); - params - } - - const TARGETS: [(f64, f64); 6] = [ - (0.01, 0.01), - (0.02, 0.01), - (0.05, 0.01), - (0.05, 0.001), - (0.1, 0.05), - (0.2, 0.2), - ]; - - /// Each L2 shape meets its target as a relative error, and one fewer - /// row or column does not. - #[test] - fn l2_shape_is_the_smallest_meeting_its_target() { - for (eps, delta) in TARGETS { - let params = size_params(SketchAlgorithm::UnivMon, &l2_intent(), eps, delta); - let guarantee = l2(¶ms).expect("an L2 guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!( - meets(&guarantee, eps, Some(delta)), - "({eps}, {delta}): {params:?}" - ); - let fewer_rows = with(¶ms, |_, rows, _| *rows -= 1); - assert!( - l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, Some(delta))), - "({eps}, {delta}): {fewer_rows:?}" - ); - let fewer_cols = with(¶ms, |_, _, cols| *cols -= 1); - assert!( - l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, Some(delta))), - "({eps}, {delta}): {fewer_cols:?}" - ); - } - } - - /// The general collision bound `9/(8w)` applies off powers of two: a - /// width just past a power of two is worse than that power of two. - #[test] - fn l2_error_uses_the_general_collision_bound_off_powers_of_two() { - let error = |cols| l2_relative_error(cols).unwrap(); - assert!(error(1 << 12) < error((1 << 12) + 1)); - let f2_error = |cols: u32| 1.0 - (1.0 - error(cols)).powi(2); - assert!((f2_error(1 << 12) - (20.0 / 4096.0f64).sqrt()).abs() < 1e-12); - assert!((f2_error(3000) - (22.5 / 3000.0f64).sqrt()).abs() < 1e-12); - } - - /// The L2 failure probability is the exact binomial tail of the rows. - #[test] - fn l2_failure_probability_is_the_binomial_tail() { - // P(Bin(1, 0.1) ≥ 1) and P(Bin(3, 0.1) ≥ 2) = 3·0.01·0.9 + 0.001. - assert!((l2_failure_probability(1) - 0.1).abs() < 1e-12); - assert!((l2_failure_probability(3) - 0.028).abs() < 1e-12); - // An even count needs half the rows to miss: worse than one fewer. - assert!(l2_failure_probability(4) > l2_failure_probability(3)); - } - - /// Shapes past the hash budget or the layer limit certify nothing, and - /// L2 sizing never builds one: a target it cannot reach gets the - /// largest shape it can build, whose guarantee misses it. - #[test] - fn l2_shapes_stay_within_the_hash_budget() { - // 8 · (16 + 1) = 136 > 128. - assert!(l2(&univmon(1, 8, 1 << 16, 1)).is_none()); - assert!(l2(&univmon(1, 7, 1 << 16, 1)).is_some()); - assert!(l2(&univmon(1, 5, 1024, 20)).is_none()); - for (eps, delta) in [(0.001, 1e-6), (1e-5, 0.01), (0.01, 1e-12)] { - let params = size_params(SketchAlgorithm::UnivMon, &l2_intent(), eps, delta); - let SketchParams::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - unreachable!() - }; - assert!( - hash_independent(sketch_rows, sketch_cols, layers), - "{params:?}" - ); - assert!( - !meets(&l2(¶ms).unwrap(), eps, Some(delta)), - "{params:?}" - ); - } - } - - /// The executor builds at most 20 rows, the rows a UnivMon state - /// encodes, so a 21-row UnivMon certifies nothing even though its hashes - /// fit the budget (21 · (5 + 1) = 126 ≤ 128 at 32 columns). - #[test] - fn univmon_past_twenty_rows_is_not_hash_independent() { - assert!(hash_independent(20, 32, 1)); - assert!(!hash_independent(21, 32, 1)); - assert!(l2(&univmon(1, 20, 32, 1)).is_some()); - assert!(l2(&univmon(1, 21, 32, 1)).is_none()); - } - - /// Each distinct-count heap meets its target once the source's bound - /// is bound, and one fewer key does not. - #[test] - fn distinct_heap_is_the_smallest_meeting_its_target() { - for (eps, delta) in TARGETS { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), eps, delta); - let bind = |params: &SketchParams, max_distinct| { - let guarantee = - distinct_guarantee(params, delta, "m").expect("a distinct guarantee"); - guarantee.bind(&|statistic| { - let (k, layers, source) = parse_incomplete_top_layer_statistic(statistic)?; - (source == "m") - .then(|| incomplete_top_layer_probability(k, layers, max_distinct)) - }) - }; - for max_distinct in [9, 10_000_000] { - let guarantee = bind(¶ms, max_distinct); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!( - meets(&guarantee, eps, Some(delta)), - "({eps}, {delta}): {params:?}" - ); - let smaller = with(¶ms, |heap, _, _| *heap -= 1); - assert!( - !meets(&bind(&smaller, max_distinct), eps, Some(delta)), - "({eps}, {delta}): {smaller:?}" - ); - } - } - } - - /// Without the source's bound the failure probability stays unknown - /// and no target is met; a bound the pyramid is too shallow for gives - /// probability 1. - #[test] - fn distinct_guarantee_needs_the_source_bound_and_enough_layers() { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - let guarantee = distinct_guarantee(¶ms, 0.01, "m").unwrap(); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(!meets(&guarantee, 0.05, Some(0.01))); - let SketchParams::UnivMon { heap_size: k, .. } = params else { - unreachable!() - }; - // At most k keys: complete for certain. - assert_eq!(incomplete_top_layer_probability(k, 2, u64::from(k)), 0.0); - // F0_max = k · 2^(L−2) is the deepest bound L layers cover. - let deepest = u64::from(k) << 17; - assert!(incomplete_top_layer_probability(k, 19, deepest) < 1e-100); - assert_eq!(incomplete_top_layer_probability(k, 19, deepest + 1), 1.0); - assert_eq!(incomplete_top_layer_probability(k, 18, deepest), 1.0); - assert_eq!( - incomplete_top_layer_probability(k, 1, u64::from(k) + 1), - 1.0 - ); - assert_eq!(incomplete_top_layer_probability(8, 19, u64::MAX), 1.0); - } - - /// Only a heap of at least 8 and a buildable shape gets a distinct - /// guarantee; the guarantee without a source is none. - #[test] - fn distinct_guarantee_needs_a_heap_of_eight_and_a_buildable_shape() { - assert!(distinct_guarantee(&univmon(7, 1, 1, 19), 0.5, "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 20), 0.5, "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 19), 0.5, "m").is_some()); - let sized = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - assert!(guarantee( - &SketchAlgorithm::UnivMon, - &sized, - &SketchStatistic::Cardinality - ) - .is_none()); - } - - /// The statistic names its heap, layers and source, any source name. - #[test] - fn incomplete_top_layer_statistics_round_trip() { - for source in ["flows_src_ip", "a[b]", "x])(", ""] { - let statistic = incomplete_top_layer_statistic(337_500, 19, source); - assert_eq!( - parse_incomplete_top_layer_statistic(&statistic), - Some((337_500, 19, source)) - ); - } - assert_eq!( - parse_incomplete_top_layer_statistic("max_distinct[m]"), - None - ); - } - - /// The total stays exact; entropy stays uncertified. - #[test] - fn total_is_exact_and_entropy_uncertified() { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - let total = guarantee( - &SketchAlgorithm::UnivMon, - ¶ms, - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }, - ) - .unwrap(); - assert!(total.is_exact()); - assert!(guarantee( - &SketchAlgorithm::UnivMon, - &UNIVMON_BASELINE, - &SketchStatistic::FrequencyEntropy - ) - .is_none()); - } -} diff --git a/crates/accuracy/src/guarantee/univmon/mod.rs b/crates/accuracy/src/guarantee/univmon/mod.rs new file mode 100644 index 000000000..2247dd74e --- /dev/null +++ b/crates/accuracy/src/guarantee/univmon/mod.rs @@ -0,0 +1,104 @@ +//! UnivMon's readouts and the guarantees `asap_sketchlib`'s standard-mode +//! estimators carry (the derivations are in the library's +//! `docs/api/api_univmon.md`, the bounds `estimate_l2` and +//! `estimate_cardinality` cite): +//! - the total (a bare `PointCount`) is the exact unit-update count; +//! - L2 reads layer 0's Count Sketch directly: per row, Chebyshev with +//! `Var ≤ 2q F2²`, and the median of the rows fails only when at least +//! `⌈r/2⌉` rows fail, an exact binomial tail; +//! - the distinct count has no guarantee here: it needs the source's +//! distinct bound; +//! - entropy has no certified bound. +//! +//! Every bound assumes the key hash is uniformly random and the shape keeps +//! the hashes independent (`hash_independent`). Terminal-mode states +//! (OctoSketch aggregators fed promoted deltas) are not covered: the +//! executor rejects them. +use super::*; + +/// The per-row failure probability the L2 guarantee fixes: a row misses by +/// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it +/// lets a shape state one `(ε, δ)`: the columns give `ε`, the rows `δ`. +const L2_ROW_FAILURE: f64 = 0.1; + +pub(super) fn guarantee( + params: &SketchParams, + query: &SketchStatistic, +) -> Option { + let SketchParams::UnivMon { + sketch_rows, + sketch_cols, + layers, + .. + } = params + else { + return None; + }; + match query { + SketchStatistic::PointCount { value: None, .. } => Some(AccuracyGuarantee::exact()), + SketchStatistic::FrequencyL2 if hash_independent(*sketch_rows, *sketch_cols, *layers) => { + Some(super::bounded_guarantee( + AccuracyMetric::Relative, + l2_relative_error(*sketch_cols)?, + ProbabilityExpr::Constant { + value: l2_failure_probability(*sketch_rows), + }, + )) + } + // The distinct count needs the source's distinct bound; entropy has + // no certified bound. + _ => None, + } +} + +/// Whether the executor builds this shape, with independent hashes: at +/// most `UNIVMON_MAX_LAYERS` (the library's) layers and `MATRIX_MAX_ROWS` (20) rows, the rows a +/// UnivMon state encodes, and `rows · (⌈log2 cols⌉ + 1) ≤ 128` +/// ([`asap_sketchlib::UnivMon::check_shape`]; the heap size is not part of +/// the hash rules). +fn hash_independent(rows: u32, cols: u32, layers: u8) -> bool { + rows as usize <= asap_sketchlib::MATRIX_MAX_ROWS + && asap_sketchlib::UnivMon::check_shape( + 1, + rows as usize, + cols as usize, + usize::from(layers), + ) + .is_ok() +} + +/// The relative error of the L2 norm at `cols` columns. Per row, the +/// squared norm misses by more than `ε₂ F2` with probability at most +/// `2q/ε₂²` ([`column_collision`]), so fixing that at +/// [`L2_ROW_FAILURE`] gives `ε₂ = √(2q/0.1)`. The norm then misses by at +/// most `1 − √(1 − ε₂)` (the low side; the high side `√(1 + ε₂) − 1` is +/// smaller). `None` when `ε₂ ≥ 1`. +fn l2_relative_error(cols: u32) -> Option { + let f2_error = (2.0 * column_collision(cols) / L2_ROW_FAILURE).sqrt(); + (f2_error < 1.0).then(|| 1.0 - (1.0 - f2_error).sqrt()) +} + +/// `P(c_i = c_j)` for two keys in a row of a UnivMon layer of `cols` +/// columns. `asap_sketchlib` takes a UnivMon column as a `⌈log2 cols⌉`-bit +/// slice of one 128-bit hash modulo `cols` ([`hash_independent`]): uniform +/// for a power of two, and otherwise at most `9/(8 cols)` (the slice values +/// past `cols` fold onto the low columns; the worst case, +/// `cols = ¾ · 2^⌈log2 cols⌉`, doubles a third of them). A standalone +/// Count Sketch hashes modulo `cols` instead (`count_sketch`). +fn column_collision(cols: u32) -> f64 { + let cols = cols.max(1); + if cols.is_power_of_two() { + 1.0 / f64::from(cols) + } else { + 9.0 / (8.0 * f64::from(cols)) + } +} + +/// The median of `rows` independent rows misses only when at least +/// `⌈rows/2⌉` of them miss: `P(Binomial(rows, 0.1) ≥ ⌈rows/2⌉)`. +fn l2_failure_probability(rows: u32) -> f64 { + super::binomial_upper_tail(rows, L2_ROW_FAILURE, rows.div_ceil(2)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/univmon/tests.rs b/crates/accuracy/src/guarantee/univmon/tests.rs new file mode 100644 index 000000000..c262eef7a --- /dev/null +++ b/crates/accuracy/src/guarantee/univmon/tests.rs @@ -0,0 +1,75 @@ +use super::*; + +/// Layers `asap_sketchlib` builds. +const MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; + +/// A shape for readouts without a size model: the total (exact +/// whatever the shape) and entropy (uncertified). +const BASELINE: SketchParams = SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1024, + layers: 16, +}; + +fn univmon(heap_size: u32, rows: u32, cols: u32, layers: u8) -> SketchParams { + SketchParams::UnivMon { + heap_size, + sketch_rows: rows, + sketch_cols: cols, + layers, + } +} + +fn l2(params: &SketchParams) -> Option { + guarantee(params, &SketchStatistic::FrequencyL2) +} + +/// The general collision bound `9/(8w)` applies off powers of two: a +/// width just past a power of two is worse than that power of two. +#[test] +fn l2_error_uses_the_general_collision_bound_off_powers_of_two() { + let error = |cols| l2_relative_error(cols).unwrap(); + assert!(error(1 << 12) < error((1 << 12) + 1)); + let f2_error = |cols: u32| 1.0 - (1.0 - error(cols)).powi(2); + assert!((f2_error(1 << 12) - (20.0 / 4096.0f64).sqrt()).abs() < 1e-12); + assert!((f2_error(3000) - (22.5 / 3000.0f64).sqrt()).abs() < 1e-12); +} + +/// The L2 failure probability is the exact binomial tail of the rows. +#[test] +fn l2_failure_probability_is_the_binomial_tail() { + // P(Bin(1, 0.1) ≥ 1) and P(Bin(3, 0.1) ≥ 2) = 3·0.01·0.9 + 0.001. + assert!((l2_failure_probability(1) - 0.1).abs() < 1e-12); + assert!((l2_failure_probability(3) - 0.028).abs() < 1e-12); + // An even count needs half the rows to miss: worse than one fewer. + assert!(l2_failure_probability(4) > l2_failure_probability(3)); +} + +/// The executor builds at most 20 rows, the rows a UnivMon state +/// encodes, so a 21-row UnivMon certifies nothing even though its hashes +/// fit the budget (21 · (5 + 1) = 126 ≤ 128 at 32 columns). +#[test] +fn univmon_past_twenty_rows_is_not_hash_independent() { + assert!(hash_independent(20, 32, 1)); + assert!(!hash_independent(21, 32, 1)); + assert!(l2(&univmon(1, 20, 32, 1)).is_some()); + assert!(l2(&univmon(1, 21, 32, 1)).is_none()); +} + +/// The total stays exact; entropy stays uncertified. +#[test] +fn total_is_exact_and_entropy_uncertified() { + let params = univmon(54_000, 1, 1, MAX_LAYERS); + let total = guarantee( + ¶ms, + &SketchStatistic::PointCount { + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, + value: None, + }, + ) + .unwrap(); + assert_eq!(total.spec.error.evaluate(), Some(0.0)); + assert_eq!(total.spec.failure_probability.evaluate(), Some(0.0)); + assert!(guarantee(&BASELINE, &SketchStatistic::FrequencyEntropy).is_none()); +} diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 13be59fc7..abf154e7d 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -1,37 +1,31 @@ //! `asap-accuracy` — the accuracy rules the planning stages share: the -//! planner's built-in analytical rules (estimators, composition, guarantee -//! propagation through a DAG, sizing, budget split and a target's comparison -//! limit [`target_limit`]), the deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ -//! of a logical edge ([`producer_satisfies_logical_requirements`]). -//! -//! Estimator models derive theoretical guarantees, composition propagates -//! them, `propagation` composes them through every node of a DAG, `sizing` -//! sizes a summary for its target, the budget split sizes each approximate -//! input of a composition for an equal share of its target, and -//! `propagation_stats` holds the scoped contracts and statistics the rules -//! consult. `guarantee` derives each summary type's theoretical guarantee, -//! `evidence` reads the deployment's measured accuracy, `requirements` -//! decides $\alpha^{\log}$ and `error_propagation` bounds a consumer's -//! output error by its inputs'. +//! theoretical guarantee of reading a statistic from a summary with given +//! parameters ([`theoretical_guarantee`], from the per-algorithm +//! guarantees in `guarantee`), the deployment's measured accuracy +//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract +//! ([`ErrorContract`], in `error_contract`), the error metrics of a memo +//! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the +//! metric of a readout's error from `readout`), each readout's required +//! accuracy ([`readout_levels`], in `readout_caps`), and $\alpha^{\log}$ of +//! a logical edge ([`ErrorMetricsOfMemo::edge_meets_logical_requirements`], +//! in `logical_edge`, with a bound's comparison to a limit +//! [`within_limit`]). //! See `docs/design_docs/concepts/accuracy-models.md` for the design. //! //! It depends only on `asap-types`, never on a stage crate, so every stage //! can use it. -mod budget; -pub mod composition; -mod error_propagation; +mod error_contract; mod evidence; -mod function_rules; -pub mod guarantee; -pub mod propagation; -mod propagation_stats; -mod requirements; -pub mod sizing; +mod guarantee; +mod logical_edge; +mod metric_rule; +mod readout; +mod readout_caps; -pub use budget::per_input_target; -pub use error_propagation::{error_propagation, ErrorPropagation}; +pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; -pub(crate) use propagation_stats::PropagationStats; -pub use requirements::{producer_satisfies_logical_requirements, target_limit}; +pub use logical_edge::within_limit; +pub use metric_rule::ErrorMetricsOfMemo; +pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/logical_edge/mod.rs b/crates/accuracy/src/logical_edge/mod.rs new file mode 100644 index 000000000..c99686394 --- /dev/null +++ b/crates/accuracy/src/logical_edge/mod.rs @@ -0,0 +1,179 @@ +//! $\alpha^{\log}$ of a logical edge +//! ([`ErrorMetricsOfMemo::edge_meets_logical_requirements`]), and the +//! comparison of an evaluated bound with a required limit +//! ([`within_limit`]). + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; +use asap_types::summary_type::{MergedError, SketchParams, SketchStatistic, SummaryTypeCapability}; + +use crate::guarantee::{guarantee_before_failure_probability_check, holds_at_failure_probability}; +use crate::readout::sketch_readout; +use crate::{AccuracyEvidence, ErrorMetricsOfMemo}; + +/// Small relative tolerance for comparing an evaluated bound against a +/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected +/// by floating-point noise. +const SATISFACTION_TOLERANCE: f64 = 1e-9; + +/// Whether an `evaluated` error (or failure probability) meets a required +/// `limit`: at most `limit` widened by [`SATISFACTION_TOLERANCE`], so a +/// bound equal to it up to floating-point noise meets it. +pub fn within_limit(evaluated: f64, limit: f64) -> bool { + evaluated <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON +} + +impl ErrorMetricsOfMemo { + /// $\alpha^{\log}$ of one edge (concepts doc, section 2): `Ok` when + /// `producer` may feed input `input_index_of_consumer` of `consumer`, + /// else why not, for Pass 2's pruned list. `self` is + /// [`ErrorMetricsOfMemo::of`] `memo`. + /// - A readout's input (`SummaryEstimate`) from a sketch state: the + /// readout's error metric is one its output allows (the metric rule; + /// no rule where no query with a target reaches it; `NoGuarantee` + /// when, under such a query, the readout has no metric or its + /// guarantee's failure probability is unknown, as HLL's), and the + /// guarantee of the producer's θ for the readout's statistic, at the + /// required failure probability, is within the required error + /// ([`within_limit`]) and in its metric: the deployment's measurement + /// (`evidence`) when it has one, else the theoretical guarantee; a + /// merged state (any producer but a build) must keep the guarantee + /// (`MergedError::Exact` or `AsIfBuiltOverUnion`). A top-k's heap + /// holds its `k` whatever the requirement. + /// - Any other input: the metric rule, when the producer is a sketch + /// readout: the consumer allows the readout's error metric on that + /// input. Only readouts introduce error, so an edge from any other + /// producer holds; a consumer reading an approximate value through it + /// is cut below, on the readout's edge. + pub fn edge_meets_logical_requirements( + &self, + memo: &Memo, + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input_index_of_consumer: usize, + evidence: &dyn AccuracyEvidence, + ) -> Result<(), RootPruneReason> { + let consumer_candidate = memo.candidate(consumer); + let input = input_index_of_consumer; + let metric_rule = |producer_metric: AccuracyMetric, allowed: &[AccuracyMetric]| { + if allowed.contains(&producer_metric) { + Ok(()) + } else { + Err(RootPruneReason::MetricRule { + consumer, + input, + producer_metric, + accepted_metrics: allowed.to_vec(), + }) + } + }; + let Operator::ASAP(ASAPOp::SummaryEstimate { .. }) = consumer_candidate.expression.operator + else { + let Some(producer_metric) = self.readout_error_metric(producer) else { + return Ok(()); + }; + return match self.error_metrics_allowed_on_inputs(consumer) { + Some(allowed) => metric_rule(producer_metric, &allowed), + None => Ok(()), + }; + }; + let producer_candidate = memo.candidate(producer); + let (Some((summary_type, statistic)), Some(parameters)) = ( + sketch_readout(memo, consumer_candidate), + producer_candidate.logical_properties.parameters(), + ) else { + // An exact state reads exactly. + return Ok(()); + }; + let summary_type = &summary_type; + if let (SketchStatistic::TopK { k }, Some(heap_size)) = (statistic, heap_size(parameters)) { + if (heap_size as usize) < *k { + return Err(RootPruneReason::HeapSmallerThanTopK { + consumer, + input, + k: *k, + }); + } + } + let required = consumer_candidate.input_requirements[input].required_accuracy; + let failure_probability = required.map_or(1.0, |required| required.failure_probability); + let measured = + evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); + let allowed = self.error_metrics_allowed_on_inputs(consumer); + // The theoretical guarantee before its failure-probability check, + // computed once for both checks below. + let theoretical = (measured.is_none() && (allowed.is_some() || required.is_some())) + .then(|| { + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + failure_probability, + ) + }) + .flatten(); + if let Some(allowed) = allowed { + // A query with a target reads this readout: its error needs a + // metric and a failure probability a guarantee states. + let no_guarantee = RootPruneReason::NoGuarantee { consumer, input }; + let metric = self + .readout_error_metric(consumer) + .ok_or(no_guarantee.clone())?; + metric_rule(metric, &allowed)?; + let evaluable = measured.is_some() + || theoretical.as_ref().is_some_and(|guarantee| { + guarantee.spec.failure_probability.evaluate().is_some() + }); + if !evaluable { + return Err(no_guarantee); + } + } + let Some(required) = required else { + return Ok(()); + }; + let merged = !matches!( + producer_candidate.expression.operator, + Operator::ASAP(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) + ); + let keeps = !merged + || matches!( + SummaryTypeCapability::of(summary_type).merged_error(statistic), + Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) + ); + let guarantee = measured.or_else(|| { + theoretical.filter(|guarantee| { + holds_at_failure_probability(guarantee, required.failure_probability) + }) + }); + let meets = guarantee.is_some_and(|guarantee| { + guarantee.spec.accuracy_metric == required.metric + && guarantee + .spec + .error + .evaluate() + .is_some_and(|error| within_limit(error, required.max_error)) + }); + if keeps && meets { + Ok(()) + } else { + Err(RootPruneReason::BelowRequiredAccuracy { + consumer, + input, + required, + }) + } + } +} + +/// A heap variant's heap size. +fn heap_size(parameters: &SketchParams) -> Option { + match parameters { + SketchParams::CmsWithHeap { heap_size, .. } + | SketchParams::CountSketchWithHeap { heap_size, .. } => Some(*heap_size), + _ => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/logical_edge/tests.rs b/crates/accuracy/src/logical_edge/tests.rs new file mode 100644 index 000000000..40d51baf9 --- /dev/null +++ b/crates/accuracy/src/logical_edge/tests.rs @@ -0,0 +1,147 @@ +use super::*; +use crate::NoMeasuredGuarantees; +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryUpdate}; +use asap_types::test_support::memo_builders::query; +use asap_types::test_support::memo_fixtures::scan; +use asap_types::workload::DataWorkload; + +/// A bound equal to the limit up to floating-point noise meets it; a +/// bound one part in a million above it does not. +#[test] +fn within_limit_admits_only_floating_point_noise() { + assert!(within_limit(0.01 * (1.0 + 1e-12), 0.01)); + assert!(!within_limit(0.01 * (1.0 + 1e-6), 0.01)); + assert!(within_limit(0.0, 0.0)); +} + +/// A distinct count read from HLL (relative error, failure probability +/// unknown) or UnivMon (no guarantee for it) under a query with a +/// relative target: the edge fails with `NoGuarantee`, its error would +/// reach the root unbounded; with no target reaching it, it holds. +#[test] +fn a_distinct_count_readout_without_an_evaluable_guarantee_serves_no_query_with_a_target() { + for algorithm in [SketchAlgorithm::Hll, SketchAlgorithm::UnivMon] { + let mut memo = Memo::new(&DataWorkload::default()); + let events = memo.find_or_add_group(scan("events", &[], vec![])).unwrap(); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: events, + summary_type: FieldDataType::Sketch(algorithm), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let state = memo.find_or_add_group(build.clone()).unwrap(); + let parameters = parameter_list(algorithm)[0].clone(); + let producer = memo + .add_candidate( + state, + CandidateForMemoGroup::new( + build, + Some(parameters), + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) + .unwrap(); + let readout = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Cardinality, + }); + let distinct = memo.find_or_add_group(readout.clone()).unwrap(); + let consumer = memo + .add_candidate( + distinct, + CandidateForMemoGroup::new( + readout, + None, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) + .unwrap(); + let edge = |memo: &Memo| { + let order = memo.topological_order().expect("acyclic"); + ErrorMetricsOfMemo::of(memo, &order).edge_meets_logical_requirements( + memo, + producer, + consumer, + 0, + &NoMeasuredGuarantees, + ) + }; + assert!(edge(&memo).is_ok(), "{algorithm:?}"); + query( + &mut memo, + distinct, + Some(AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative)), + ); + assert!( + matches!( + edge(&memo), + Err(RootPruneReason::NoGuarantee { input: 0, .. }) + ), + "{algorithm:?}: {:?}", + edge(&memo) + ); + } +} + +/// A top-k heap smaller than k never serves the readout, whatever +/// accuracy is required: the edge from each build into `topk(50)`'s +/// readout fails with `HeapSmallerThanTopK` exactly for the heaps +/// below 50 (sizes 10 and 20, with each width and depth). +#[test] +fn a_heap_smaller_than_k_fails_the_topk_readouts_edge() { + use asap_types::test_support::memo_builders::{ + estimate, group, own_candidate, scan_group, sketch_state, + }; + use asap_types::test_support::memo_fixtures::range; + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let state = sketch_state( + &mut memo, + window, + SketchAlgorithm::CmsWithHeap, + Reduction::by(vec![]), + ); + let readout = estimate(&mut memo, state, SketchStatistic::TopK { k: 50 }); + let error_metrics = ErrorMetricsOfMemo::of(&memo, &memo.topological_order().expect("acyclic")); + let mut small = 0; + for index_in_group in 0..memo.groups()[state.0 as usize].candidates.len() { + let build = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + let Some(SketchParams::CmsWithHeap { heap_size, .. }) = + memo.candidate(build).logical_properties.parameters() + else { + panic!("a heap build"); + }; + let edge = error_metrics.edge_meets_logical_requirements( + &memo, + build, + own_candidate(readout), + 0, + &NoMeasuredGuarantees, + ); + if *heap_size < 50 { + assert!( + matches!( + edge, + Err(RootPruneReason::HeapSmallerThanTopK { k: 50, .. }) + ), + "{edge:?}" + ); + small += 1; + } else { + assert!(edge.is_ok(), "{edge:?}"); + } + } + assert_eq!(small, 2 * 28); +} diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs new file mode 100644 index 000000000..46adafebb --- /dev/null +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -0,0 +1,280 @@ +//! The error metrics of a memo (concepts doc, section 4, "Metric rule"): +//! each candidate's error contract per metric, the metrics each group's +//! output may be off in, each sketch readout's error metric; the metric +//! rule on edges ([`ErrorMetricsOfMemo::edge_meets_logical_requirements`], +//! in `logical_edge`) and per query +//! ([`ErrorMetricsOfMemo::exact_only_groups_of_query`]). + +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::{QueryRoot, ScalarExpr}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId}; + +use crate::error_contract::{error_contract, ErrorContract}; +use crate::readout::sketch_readout_error_metric; +use crate::readout_caps::caps; + +/// How many error metrics there are: `AccuracyMetric::ALL`'s length. +pub(crate) const METRIC_COUNT: usize = AccuracyMetric::ALL.len(); + +/// Per metric, in `AccuracyMetric::ALL` order: a candidate's error +/// contract in it; `None` = no contract. +pub(crate) type ErrorContractPerMetric = [Option; METRIC_COUNT]; + +/// The error metrics of one memo state, built once per memo state before +/// an edge pass; Stage 3 reads the same contracts. +pub struct ErrorMetricsOfMemo { + /// [group][candidate][metric]: the candidate's error contract; `None` = + /// no contract. + pub(crate) error_contract: Vec>, + /// [group]: the error metrics its output may have, in + /// `AccuracyMetric::ALL` order; `None` = no query with a target reaches + /// it (no metric rule applies there). + error_metrics_allowed_at_output: Vec>>, + /// [group][candidate]: the error metric of a sketch readout; `None` = + /// not a sketch readout, or one whose summary type has no guarantee + /// for the statistic it reads. + readout_error_metric: Vec>>, +} + +impl ErrorMetricsOfMemo { + /// The error metrics of `memo`, whose topological order is `order` + /// (`Memo::topological_order`). A group's output may have the metrics + /// the queries with a target reaching it allow: a query root allows + /// its target's metrics ([`query_roots_with_target`]); a candidate's + /// input those its output allows that its error contract passes to it; + /// a group the union over its consumers. These are the metrics of the + /// readout caps' walk (`readout_caps`), which also reaches a group + /// through inputs the output does not depend on. + pub fn of(memo: &Memo, order: &[MemoGroupId]) -> Self { + let error_contract: Vec> = memo + .groups() + .iter() + .map(|group| { + group + .candidates + .iter() + .map(|candidate| { + AccuracyMetric::ALL.map(|metric| { + error_contract(memo, &candidate.expression.operator, metric) + }) + }) + .collect() + }) + .collect(); + let error_metrics_allowed_at_output = caps(memo, &error_contract, order) + .into_iter() + .map(|caps| { + caps.map(|caps| { + AccuracyMetric::ALL + .into_iter() + .filter(|metric| caps.contains_key(metric)) + .collect() + }) + }) + .collect(); + let readout_error_metric = memo + .groups() + .iter() + .map(|group| { + group + .candidates + .iter() + .map(|candidate| sketch_readout_error_metric(memo, candidate)) + .collect() + }) + .collect(); + Self { + error_contract, + error_metrics_allowed_at_output, + readout_error_metric, + } + } + + /// `candidate`'s error contract in `metric`; `None` when it has none. + pub fn error_contract( + &self, + candidate: CandidateForMemoGroupId, + metric: AccuracyMetric, + ) -> Option<&ErrorContract> { + self.error_contract[candidate.group.0 as usize][candidate.index_in_group] + [metric_index(metric)] + .as_ref() + } + + /// The error metric of `candidate` when it is a sketch readout. + pub(crate) fn readout_error_metric( + &self, + candidate: CandidateForMemoGroupId, + ) -> Option { + self.readout_error_metric[candidate.group.0 as usize][candidate.index_in_group] + } + + /// The metrics among `metrics` that `consumer`'s error contract passes + /// to its inputs: those an input of it may be off in when its output + /// may be off in `metrics`. + fn error_metrics_passed_to_inputs( + &self, + consumer: CandidateForMemoGroupId, + metrics: &[AccuracyMetric], + ) -> Vec { + metrics + .iter() + .copied() + .filter(|&metric| self.error_contract(consumer, metric).is_some()) + .collect() + } + + /// The metrics `consumer` allows an input's error in, the metric rule + /// on its input edges: those its group's output allows that its error + /// contract passes to its inputs (a readout's contract passes every + /// metric, so a readout allows its output's). `None` where no query + /// with a target reaches the consumer. + pub(crate) fn error_metrics_allowed_on_inputs( + &self, + consumer: CandidateForMemoGroupId, + ) -> Option> { + let allowed = self.error_metrics_allowed_at_output[consumer.group.0 as usize].as_ref()?; + Some(self.error_metrics_passed_to_inputs(consumer, allowed)) + } + + /// Per query: the groups below its root it may read only exactly, the + /// metric rule per query where the edges apply it per group. Walking + /// down the live candidates (not in `dead`) from each root with a + /// target, which allows its target's metrics + /// ([`query_roots_with_target`]), a candidate passes to its inputs the + /// metrics its error contract passes; a group keeps, per query, the + /// metrics every path to it passes. A group is exact-only for a query + /// when no metric reaches it (one path through a consumer without a + /// contract, a comparison or a sum, suffices) or when it holds a sketch + /// readout whose error metric is not among them (`quantile_over_time` + /// at a relative target reads no KLL readout, whose error is a rank + /// one); and so is every group below one. + /// `exact_only_readout_groups_of_query` adds, per query, the readout + /// groups the readout requirements list (a readout whose level another + /// query gave, for a query whose failure probability has no per-value + /// one). `order` is the memo's topological order. + pub fn exact_only_groups_of_query( + &self, + memo: &Memo, + order: &[MemoGroupId], + dead: &BTreeSet, + exact_only_readout_groups_of_query: &BTreeMap>, + ) -> Vec> { + let live_candidates = |group: MemoGroupId| { + (0..memo.groups()[group.0 as usize].candidates.len()) + .map(move |index_in_group| CandidateForMemoGroupId { + group, + index_in_group, + }) + .filter(|candidate| !dead.contains(candidate)) + }; + let mut allowed: Vec>> = + vec![BTreeMap::new(); memo.groups().len()]; + // Keeps, per query, the metrics every path to a group passes. + fn keep_common( + allowed: &mut [BTreeMap>], + group: MemoGroupId, + query: usize, + metrics: Vec, + ) { + allowed[group.0 as usize] + .entry(query) + .and_modify(|known| known.retain(|metric| metrics.contains(metric))) + .or_insert(metrics); + } + for (query, root, _, metrics) in query_roots_with_target(memo) { + keep_common(&mut allowed, root, query, metrics); + } + let mut exact_only = vec![BTreeSet::new(); memo.queries().len()]; + for (&query, groups) in exact_only_readout_groups_of_query { + exact_only[query].extend(groups.iter().copied()); + } + for &group in order.iter().rev() { + let allowed_here = allowed[group.0 as usize].clone(); + for (&query, metrics) in &allowed_here { + if metrics.is_empty() { + exact_only[query].insert(group); + } + } + for consumer in live_candidates(group) { + if let Some(metric) = self.readout_error_metric(consumer) { + for (&query, metrics) in &allowed_here { + if !metrics.contains(&metric) { + exact_only[query].insert(group); + } + } + } + let passed = self.error_metrics_passed_to_inputs(consumer, &AccuracyMetric::ALL); + for &child in memo.candidate(consumer).expression.operator.children() { + for (&query, metrics) in &allowed_here { + let passed_here = metrics + .iter() + .copied() + .filter(|metric| passed.contains(metric)) + .collect(); + keep_common(&mut allowed, child, query, passed_here); + } + } + } + } + // Every group below an exact-only group is read exactly too. + for &group in order.iter().rev() { + for groups in &mut exact_only { + if groups.contains(&group) { + for consumer in live_candidates(group) { + groups.extend( + memo.candidate(consumer) + .expression + .operator + .children() + .into_iter() + .copied(), + ); + } + } + } + } + exact_only + } +} + +/// The index of `metric` in `AccuracyMetric::ALL`. +fn metric_index(metric: AccuracyMetric) -> usize { + AccuracyMetric::ALL + .iter() + .position(|&listed| listed == metric) + .expect("AccuracyMetric::ALL lists every metric") +} + +/// Each query with a target, with the groups its root reads and the metrics +/// each allows: an operator root, `scalar(v)` and an SQL scalar subquery +/// (which return the value they read) allow the target's metrics +/// (`AccuracyTarget::metrics`); any other scalar expression's operands +/// allow none, as no error contract covers it, so they are read exactly. +pub(crate) fn query_roots_with_target( + memo: &Memo, +) -> Vec<(usize, MemoGroupId, &AccuracyTarget, Vec)> { + let mut roots = Vec::new(); + for (query, memo_query) in memo.queries().iter().enumerate() { + let Some(target) = &memo_query.accuracy_target else { + continue; + }; + match &memo_query.root { + QueryRoot::Operator(root) + | QueryRoot::Scalar( + ScalarExpr::PromqlScalarFromVector(root) | ScalarExpr::ScalarSubquery(root), + ) => roots.push((query, *root, target, target.metrics())), + QueryRoot::Scalar(expr) => { + for &operand in expr.operator_refs() { + roots.push((query, operand, target, Vec::new())); + } + } + } + } + roots +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/metric_rule/tests.rs b/crates/accuracy/src/metric_rule/tests.rs new file mode 100644 index 000000000..991be4279 --- /dev/null +++ b/crates/accuracy/src/metric_rule/tests.rs @@ -0,0 +1,280 @@ +use super::*; +use crate::NoMeasuredGuarantees; +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; +use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate}; +use asap_types::stages::{MemoQuery, RootPruneReason}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; +use asap_types::test_support::memo_builders::{ + estimate, group, own_candidate, quantile_over_time, quantile_over_time_readouts, query, + scan_group, QuantileReadout, QuantileReadouts, +}; +use asap_types::test_support::memo_fixtures::{arithmetic, compare, range}; +use asap_types::workload::DataWorkload; + +fn error_metrics(memo: &Memo) -> ErrorMetricsOfMemo { + ErrorMetricsOfMemo::of(memo, &memo.topological_order().expect("acyclic")) +} + +/// The metrics `group`'s output may be off in. +fn allowed_at_output(memo: &Memo, group: MemoGroupId) -> Option> { + error_metrics(memo).error_metrics_allowed_at_output[group.0 as usize].clone() +} + +/// The metrics `consumer` allows on its inputs. +fn allowed_on_inputs( + memo: &Memo, + consumer: CandidateForMemoGroupId, +) -> Option> { + error_metrics(memo).error_metrics_allowed_on_inputs(consumer) +} + +/// $\alpha^{\log}$ of the edge from `producer` into input `input` of +/// `consumer`, with the metric rule of `memo`'s queries. +fn edge( + memo: &Memo, + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input: usize, +) -> Result<(), RootPruneReason> { + error_metrics(memo).edge_meets_logical_requirements( + memo, + producer, + consumer, + input, + &NoMeasuredGuarantees, + ) +} + +/// The edge on which `readout` introduces its error into `consumer`'s +/// input `input`: from the readout candidate itself into `consumer` +/// when the quantile group reads it directly, else into the renaming +/// `Project` that reads it. +fn readout_edge( + memo: &Memo, + readout: &QuantileReadout, + consumer: CandidateForMemoGroupId, + input: usize, +) -> Result<(), RootPruneReason> { + if readout.readout == readout.candidate_of_result { + edge(memo, readout.readout, consumer, input) + } else { + edge(memo, readout.readout, readout.candidate_of_result, 0) + } +} + +/// `quantile_over_time(q, m[5m])` per series of `m` by `job` with KLL +/// and DDSketch readouts beside the exact quantile. +fn quantile_readouts(memo: &mut Memo, q: f64) -> QuantileReadouts { + let input = scan_group(memo, "m", &["job"], vec![]); + let window = group(memo, range(300_000, input)); + quantile_over_time_readouts( + memo, + window, + q, + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + ) +} + +/// A ratio reads no rank error: with a relative target, the KLL +/// readout's metric, rank, is accepted neither by the ratio nor at its +/// own output, so its edge into the division fails the metric rule. +#[test] +fn a_ratio_reads_no_rank_error() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let p50 = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.5 }); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, ratio, Some(target)); + let relative = Some(vec![AccuracyMetric::Relative]); + assert_eq!(allowed_at_output(&memo, p99), relative); + assert_eq!(allowed_on_inputs(&memo, own_candidate(ratio)), relative); + assert!(matches!( + edge(&memo, own_candidate(p99), own_candidate(ratio), 0), + Err(RootPruneReason::MetricRule { + producer_metric: AccuracyMetric::Rank, + .. + }) + )); +} + +/// The metric rule cuts edges, not candidates. +/// `avg by (job)(q0.5 / q0.5 + q0.9)` at absolute error 0.05, each +/// quantile per series with KLL, DDSketch and exact candidates: the `+` +/// carries absolute error only, so the division's inputs accept no +/// metric and the edges from the DDSketch and KLL readouts into it are +/// cut; the exact quantile's edge into the division and the division's +/// edge into the `+` hold. +#[test] +fn the_metric_rule_cuts_the_readout_edges_and_keeps_the_exact_division() { + let mut memo = Memo::new(&DataWorkload::default()); + let p50 = quantile_readouts(&mut memo, 0.5); + let window = *memo.groups()[p50.result.0 as usize].expression.children()[0]; + let p90 = quantile_over_time_readouts( + &mut memo, + window, + 0.9, + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + ); + let ratio = group( + &mut memo, + arithmetic(ArithmeticOpKind::Div, p50.result, p50.result), + ); + let sum = group( + &mut memo, + arithmetic(ArithmeticOpKind::Add, ratio, p90.result), + ); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: sum, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::ABSOLUTE); + query(&mut memo, avg, Some(target)); + let division = own_candidate(ratio); + assert_eq!(allowed_on_inputs(&memo, division), Some(vec![])); + let result = edge(&memo, division, own_candidate(sum), 0); + assert!(result.is_ok(), "{result:?}"); + for input in 0..2 { + let result = edge(&memo, own_candidate(p50.result), division, input); + assert!(result.is_ok(), "{result:?}"); + for readout in &p50.by_algorithm { + assert!( + matches!( + readout_edge(&memo, readout, division, input), + Err(RootPruneReason::MetricRule { .. }) + ), + "{:?}", + readout.algorithm + ); + } + } +} + +/// A `Filter` whose predicate reads only label columns keeps or drops +/// whole rows, so a value's error passes through it unchanged. +/// `quantile_over_time(0.99, latency[5m])` behind `job = "api"`, +/// relative error 0.05: the filter accepts relative error on its input +/// and the DDSketch readout's edge holds. +#[test] +fn a_key_only_filter_keeps_the_ddsketch_readout_admissible() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "latency", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); + let job = memo.groups()[p99.result.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == "job") + .expect("the result keeps the job label"); + let filtered = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + job, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )), + child: p99.result, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, filtered, Some(target)); + assert_eq!( + allowed_on_inputs(&memo, own_candidate(filtered)), + Some(vec![AccuracyMetric::Relative]) + ); + let result = readout_edge(&memo, &p99.by_algorithm[0], own_candidate(filtered), 0); + assert!(result.is_ok(), "{result:?}"); +} + +/// `avg` has an absolute contract only here; a relative bound on a +/// mean needs the values' signs. So +/// `avg(quantile_over_time(0.99, m[5m]))` with a relative target cuts +/// the edges from the sketch readouts into the `avg`: it reads the +/// exact quantile only. +#[test] +fn avg_of_readouts_under_a_relative_target_reads_exact_only() { + let mut memo = Memo::new(&DataWorkload::default()); + let p99 = quantile_readouts(&mut memo, 0.99); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99.result, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, avg, Some(target)); + for readout in &p99.by_algorithm { + assert!( + matches!( + readout_edge(&memo, readout, own_candidate(avg), 0), + Err(RootPruneReason::MetricRule { .. }) + ), + "a sketch readout is admissible into avg: {:?}", + readout.algorithm + ); + } + let result = edge(&memo, own_candidate(p99.result), own_candidate(avg), 0); + assert!(result.is_ok(), "{result:?}"); +} + +/// A scalar root's operands are roots: `scalar(v)` returns the value +/// it reads, so `scalar(quantile_over_time(0.99, m[5m]))` at a relative +/// target allows relative error at the quantile group, and the DDSketch +/// readout's edge holds; `scalar(v) * 2` has no error contract, so its +/// operand allows no metric and every readout edge below it is cut. +#[test] +fn a_scalar_roots_operands_are_roots() { + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + fn scalar_of(result: MemoGroupId) -> ScalarExpr { + ScalarExpr::PromqlScalarFromVector(result) + } + fn doubled(result: MemoGroupId) -> ScalarExpr { + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(scalar_of(result)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + } + } + let check = |root: fn(MemoGroupId) -> ScalarExpr, + allowed: Vec, + ddsketch_holds: bool| { + let mut memo = Memo::new(&DataWorkload::default()); + let p99 = quantile_readouts(&mut memo, 0.99); + memo.add_query(MemoQuery { + root: QueryRoot::Scalar(root(p99.result)), + accuracy_target: Some(target.clone()), + number_of_values_per_query_evaluation: Some(1), + }); + assert_eq!(allowed_at_output(&memo, p99.result), Some(allowed)); + let ddsketch = p99 + .by_algorithm + .iter() + .find(|readout| readout.algorithm == SketchAlgorithm::DDSketch) + .expect("a DDSketch readout"); + let edge = readout_edge(&memo, ddsketch, ddsketch.candidate_of_result, 0); + assert_eq!(edge.is_ok(), ddsketch_holds, "{edge:?}"); + }; + check(scalar_of, vec![AccuracyMetric::Relative], true); + check(doubled, vec![], false); +} diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs deleted file mode 100644 index 6ed6ad9e7..000000000 --- a/crates/accuracy/src/propagation.rs +++ /dev/null @@ -1,1615 +0,0 @@ -//! Guarantee propagation: the theoretical guarantee of every node of a -//! composed DAG, composed with the built-in analytical rules -//! (`crate::composition::propagate`) through operators, and the helpers -//! Pass 2 and plan selection (Stage 3) use to read, bind and recompose -//! those guarantees and to resize the summaries they depend on. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::accuracy::{ - AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, - ProbabilityExpr, -}; -use asap_types::ir::operator::arguments::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::schema::{DataType, Schema}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, - SchemaDerivationError, -}; -use asap_types::summary_type::{ - raw_data_samples_of_metric, ExactKind, MergedError, NonNegativeWeightProof, SketchAlgorithm, - SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, -}; - -use crate::composition::{ - additive_as_absolute, propagate, raw_data_samples_sign_statistic, StreamNorm, GROUP_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, -}; - -use crate::function_rules::approximate_input_composition; -use crate::PropagationStats; - -/// A rebuilt node per original node, keyed by the original's address. -type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; - -/// The approximate inputs a query's value composes: every sketch summary -/// state reachable from `root`, each once, when the value carries a -/// guarantee that is not exact ([`root_guarantee`]), including compositions -/// conditional on source-sign facts supplied later. Empty when neither the -/// current nor conditional contract composes the inputs, or the result is -/// exact. Pass 1 sizes each for its share of failure probability too. -pub fn approximate_inputs(root: &QueryRoot) -> Vec> { - // Sizing may depend on a source-domain proof supplied only at Stage 3. - // Count the inputs of that conditional composition too, without attaching - // the hypothetical proof to the candidate or accepting its accuracy. - let guarantee = root_guarantee(root).or_else(|| { - bind_guarantees(root, &|guarantee| { - guarantee.bind(&|statistic| { - crate::composition::parse_raw_data_samples_sign_statistic(statistic).map(|_| 1.0) - }) - }) - }); - if guarantee.is_none_or(|g| g.is_exact()) { - return Vec::new(); - } - let operators = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - let mut seen = HashSet::new(); - operators - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - summary_type: FieldDataType::Sketch(..), - .. - } - ) - ) - }) - .filter(|node| seen.insert(Rc::as_ptr(node))) - .collect() -} - -/// The theoretical guarantee of a query's output, as composed: the root -/// node's own, or a scalar root's composed over the operators it reads -/// (`scalar_guarantee`). -pub fn root_guarantee(root: &QueryRoot) -> Option { - match root { - QueryRoot::Operator(node) => node.guarantee.clone(), - QueryRoot::Scalar(expr) => scalar_guarantee(expr, &|_| None), - } -} - -/// Bind known statistics in existing contracts, then recover compositions -/// that previously required constant child bounds. Physical state contracts, -/// schemas and sharing are preserved. This does not -/// infer facts: `bind` must leave unsupported statistics unknown. -pub fn bind_guarantees( - root: &QueryRoot, - bind: &dyn Fn(&AccuracyGuarantee) -> AccuracyGuarantee, -) -> Option { - fn walk( - node: &Rc, - bind: &dyn Fn(&AccuracyGuarantee) -> AccuracyGuarantee, - cache: &mut NodeToRewrittenNodeCache, - ) -> Rc { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return done.clone(); - } - let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); - rebuilt.guarantee = node - .guarantee - .as_ref() - .map(bind) - .or_else(|| composed_guarantee(&rebuilt).as_ref().map(bind)); - let rebuilt = Rc::new(rebuilt); - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - rebuilt - } - let mut cache = HashMap::new(); - let bound = match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, bind, &mut cache)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| walk(node, bind, &mut cache))) - } - }; - root_guarantee(&bound) -} - -/// The one source a summary state counts the distinct items of its own -/// rows of, as `source_distinct_guarantee` reads it: the source whose -/// distinct bound gives its failure probability. `None` for a state over -/// computed rows or several sources. -pub fn distinct_bound_source(state: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }) = - state.asap() - else { - return None; - }; - source_scan(child)?; - source(child) -} - -/// `roots` with every node's guarantee composed again -/// ([`composed_guarantee`]), children first; a sub-DAG shared between roots -/// stays shared. Pass 2 calls it after re-sizing a shared summary, so each -/// consumer's guarantee is the shared state's. -pub fn recompose_guarantees(roots: &[QueryRoot]) -> Result, SchemaDerivationError> { - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(with_guarantees(node, &mut cache)?) - } - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - with_guarantees(node, &mut cache)?; - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), - ) - } - }) - }) - .collect() -} - -/// `node` and everything below it, each node with its [`composed_guarantee`]. -pub fn with_guarantees( - node: &Rc, - cache: &mut NodeToRewrittenNodeCache, -) -> Result, SchemaDerivationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - with_guarantees(child, cache)?; - } - let mut rebuilt = node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?; - rebuilt.guarantee = composed_guarantee(&rebuilt); - let rebuilt = Rc::new(rebuilt); - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) -} - -/// An extremum over at most one row per group is an identity on its value: -/// an aggregate whose measures are all `MIN`/`MAX` over a (possibly -/// filtered) grouped estimate, where every column of a unique key of the -/// estimate is a group key or pinned to one value by the filter -/// (Pass 2's finer-aggregate same-level read). Unlike a general -/// aggregate over rank estimates, this does not change the population to -/// which the input's error bound refers. -fn selected_group_guarantee(node: &OperatorNode) -> Option { - let NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - filters, - having: None, - child, - .. - } = node.non_asap()? - else { - return None; - }; - if by.is_without() - || !filters.iter().all(Option::is_none) - || measures.is_empty() - || !measures.iter().all(|measure| { - matches!( - measure, - AggIntent::Min { col: Some(_) } | AggIntent::Max { col: Some(_) } - ) - }) - { - return None; - } - let (pred, grouped) = match child.non_asap()? { - NonASAPOp::Filter { pred, child } => (Some(&pred.0), child), - _ => (None, child), - }; - let pinned = pred.map_or_else(Vec::new, pinned_columns); - // The filter may read only key columns: a predicate on an estimate - // would decide by an approximate value which groups are kept. - let reads = pred.map_or_else(Vec::new, ScalarExpr::columns_referenced); - if !grouped.schema.unique_keys.iter().any(|key| { - key.iter() - .all(|column| by.keys().contains(column) || pinned.contains(column)) - && reads.iter().all(|column| key.contains(column)) - }) { - return None; - } - let mut source = grouped; - while let Some(NonASAPOp::Project { cols, child, .. }) = source.non_asap() { - if !cols - .iter() - .enumerate() - .all(|(index, item)| matches!(item.expr, ScalarExpr::Column(column) if column == index)) - { - return None; - } - source = child; - } - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = source.asap() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(keys), - child: raw, - .. - }) = summary_input.asap() - else { - return None; - }; - if keys.is_without() - || !raw - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - { - return None; - } - grouped.guarantee.clone() -} - -/// The theoretical guarantee of a composed node, given its children's, -/// composed with the built-in analytical rules (`propagate`); -/// Stage 1 reads no data statistics or deployment evidence. Exact over exact -/// inputs; a summary estimate has its sized family's bound when -/// `estimate_has_bound`, composed with its input's when that is -/// approximate; an exact aggregation or accumulator over approximate values -/// uses its function's rule (`approximate_input_composition`); arithmetic composes as -/// `arithmetic_guarantee`. A rule that needs a statistic leaves it -/// symbolic (`BoundExpr::Unknown`), and plan selection (Stage 3) binds it. -/// `None` where no rule applies: a comparison, a value `HAVING` or any other -/// predicate over an approximate value, a rank error through any operator (a -/// rank interval counts as none in the MVP). A projection of columns only -/// keeps its child's guarantee. A summary state carries none itself. -/// -/// A guarantee holds per output point (per series or group), with its -/// failure probability per point ([`AccuracyScope::PerValue`]). Plan -/// selection states it over a whole result of `G` points as `G·δ`. -/// -/// [`AccuracyScope::PerValue`]: asap_types::accuracy::AccuracyScope::PerValue -pub fn composed_guarantee(node: &OperatorNode) -> Option { - if let Some(guarantee) = selected_group_guarantee(node) { - return Some(guarantee); - } - if let Some(guarantee) = rolled_up_guarantee(node) { - return guarantee; - } - if let Some(guarantee) = merged_tumbling_windows_guarantee(node) { - return guarantee; - } - let exact = |node: &OperatorNode| { - node.guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - }; - // A summary state's input rows and any subquery in its filter. - let exact_inputs = |state: &OperatorNode| state.children().into_iter().all(|c| exact(c)); - // Over approximate input rows, a summary's filter must not decide which - // rows count by an estimate: it may read no input column (guarantees are - // per node, so any column may carry the estimate; column-level tracking - // is #689), and its subqueries must be exact. - let exact_filter = |state: &OperatorNode| { - let Some(ASAPOp::SummaryAgg { filter, .. }) = state.asap() else { - return false; - }; - filter - .as_ref() - .is_none_or(|filter| !reads_columns(&filter.0)) - && state.children().into_iter().skip(1).all(|c| exact(c)) - }; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(..), - .. - }) if exact_inputs(child) => Some(AccuracyGuarantee::exact( - "exact accumulator over exact input", - )), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(kind, _), - child: input, - reduction, - .. - }) if exact_filter(child) && !groups_by_estimate(reduction, input) => { - let stats = fold_stats(reduction, input)?; - let input = operand_guarantee(input)?; - let op = match kind { - // Counts rows, not values. - ExactKind::Count => return Some(AccuracyGuarantee::exact("count of rows")), - ExactKind::Sum => CompositionOperator::ExactSum, - ExactKind::Min | ExactKind::Max => CompositionOperator::ExactExtremum, - ExactKind::Rate => CompositionOperator::CounterRate, - ExactKind::IRate => CompositionOperator::InstantCounterRate, - ExactKind::Increase => CompositionOperator::CounterIncrease, - }; - propagate(&op, &[input], None, &stats).ok() - } - _ => None, - }, - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => match summary_input.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: family, - input: update, - child: input, - reduction, - .. - }) if estimate_has_bound(family, update, query) - || estimate_bound_over_raw_data_samples(family, update, query).is_some() => - { - let local = match estimate_bound_over_raw_data_samples(family, update, query) { - Some(gated) => gated, - None => source_distinct_guarantee(family, query, input) - .or_else(|| { - todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters") - })?, - }; - let local = interpolation_guarantee(family, update, query, input, local)?; - if exact_inputs(summary_input) { - Some(local) - } else if exact_filter(summary_input) && !groups_by_estimate(reduction, input) { - compose( - &CompositionOperator::ApproximateAggregate, - vec![input.guarantee.clone()?], - Some(&local), - ) - } else { - None - } - } - _ => None, - }, - Operator::ASAP(_) => None, - // Renaming or reordering columns keeps the child's guarantee. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - child.guarantee.clone() - } - // Each step re-evaluates the child: every sample keeps its guarantee. - Operator::NonASAP(NonASAPOp::PromqlSubquery { child, .. }) => child.guarantee.clone(), - Operator::NonASAP(_) if node.children().into_iter().all(|child| exact(child)) => { - Some(AccuracyGuarantee::exact("exact over exact inputs")) - } - // Over a join, the child's sample value may be an exact input's - // column, so each column's own guarantee is read instead. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) if derives_from_join(child) => { - relational_project_guarantee(cols, child) - } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => project_guarantee(cols, child), - Operator::NonASAP(NonASAPOp::Join { .. }) => join_guarantee(node), - // A measure filter or `HAVING` over approximate values changes - // which rows or groups count: no rule. - Operator::NonASAP(NonASAPOp::Aggregate { - measures, - filters, - having: None, - reduction, - child, - .. - }) if filters.iter().all(Option::is_none) && !groups_by_estimate(reduction, child) => { - match measures.as_slice() { - [intent] => propagate( - &approximate_input_composition(intent)?, - &[operand_guarantee(child)?], - None, - &fold_stats(reduction, child)?, - ) - .ok(), - _ => None, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { - operator: - BinaryOperator { - kind: BinaryOpKind::Arithmetic(op), - .. - }, - lhs, - rhs, - .. - }) => arithmetic_guarantee( - op.clone(), - (operand_guarantee(lhs), None), - (operand_guarantee(rhs), None), - Rc::ptr_eq(lhs, rhs), - ), - Operator::NonASAP(_) => None, - } -} - -/// The guarantee of an estimate or finalized value read from a rollup (a -/// regrouping `SummaryMerge` of one input): the same read of the input -/// states, when the family's merged error for that read is exact or the -/// same as one build's over the union (the family-capability table). The -/// groups of one build partition its rows, so a rolled-up state is a merge -/// of disjoint parts, and a family whose bound holds for one build over -/// the union keeps it: KMV, HLL and DDSketch merge into the state a direct -/// coarse build would have, KLL into a state with the same rank bound, and -/// linear sketches into the direct build's counters, whose error is -/// relative to the coarse group's total. Each guarantee holds per output -/// point, so the number of groups does not change it. None when the merged -/// readout is approximate (heap top-k, entropy) or the family has no merge. -/// `None` for any node that does not read a rollup. -fn rolled_up_guarantee(node: &OperatorNode) -> Option> { - let state = match node.asap()? { - ASAPOp::SummaryEstimate { summary_input, .. } => summary_input, - ASAPOp::FinalizeExactAccumulator { child } => child, - _ => return None, - }; - let Some(ASAPOp::SummaryMerge { - children, - group_by: Some(_), - .. - }) = state.asap() - else { - return None; - }; - let [input] = children.as_slice() else { - return Some(None); - }; - // Kept whole, a selected state keeps the guarantee it was built with. - let input = - asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); - let family = state.asap()?.produced_state()?; - let merged = match node.asap()? { - ASAPOp::SummaryEstimate { query, .. } => { - SummaryTypeCapability::of(family).merged_error(query) - } - // An exact accumulator merged across groups is exact. - _ => SummaryTypeCapability::merges_groups(family).then_some(MergedError::Exact), - }; - if !matches!( - merged, - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) { - return Some(None); - } - let consumer = OperatorNode::with_schema( - node.operator.map_children(|child| { - if Rc::ptr_eq(child, state) { - Rc::clone(&input) - } else { - Rc::clone(child) - } - }), - node.schema.clone(), - ); - Some(composed_guarantee(&consumer)) -} - -/// The guarantee of an estimate or finalized value read from merged -/// tumbling windows (a state whose [`OperatorNode::summary_build`] is a -/// chain): the guarantee of the same read of one build over the time range -/// the merged windows cover when the -/// summary type's merged error for that read is exact or the same as that build's -/// (the summary-type capability table); none when it is approximate (evidence -/// only, for plan selection to judge) or the family has no merge. `None` for -/// any node that does not read merged tumbling windows. -fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let (state, query) = match node.asap()? { - ASAPOp::SummaryEstimate { - summary_input, - query, - } => (summary_input, Some(query)), - ASAPOp::FinalizeExactAccumulator { child } => (child, None), - _ => return None, - }; - let ASAPOp::SummaryTumblingWindowChain { summary_type, .. } = state.summary_build()? else { - return None; - }; - let capability = SummaryTypeCapability::of(&summary_type); - let merged = match query { - Some(query) => capability.merged_error(query), - // An exact accumulator's merged state is exact. - None => capability.merge_inputs.map(|_| MergedError::Exact), - }; - if !matches!( - merged, - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) { - return Some(None); - } - todo!( - "#1207 PR 4: the guarantee of a read of merged tumbling windows, as of one build over their window, over memo groups" - ) -} - -/// The guarantee of a distinct count of one instance per group over a -/// source's own rows, whose failure probability is left symbolic in the -/// source's distinct bound, which Stage 1 does not read: -/// - classic HLL's ([`hll::classic_guarantee`]); -/// - UnivMon's, sized for a distinct count ([`univmon::distinct_guarantee`]). -/// -/// `None` for any other summary, or one over computed rows, whose distinct -/// items the source's bound does not bound. -pub(crate) fn source_distinct_guarantee( - family: &FieldDataType, - query: &SketchStatistic, - input: &Rc, -) -> Option { - let FieldDataType::Sketch(algorithm) = family else { - return None; - }; - if *query != SketchStatistic::Cardinality { - return None; - } - source_scan(input)?; - match algorithm { - SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { - todo!( - "#1207 PR 4: the distinct-count guarantee of θ from LogicalProperties::parameters" - ) - } - _ => None, - } -} - -/// The rows one group of an exact fold over approximate `input` counts. -/// Across PromQL series (`Reduce`) they are at most the input series, an -/// input-row-count statistic left symbolic for plan selection; across SQL -/// groups they are a group-row-count statistic nothing binds; over time -/// (`PerEntity`) they are one series' samples in the window: the steps of a -/// subquery with an explicit resolution, `⌈range / resolution⌉`. `None` -/// when that count is not in the query. -pub(crate) fn fold_stats( - reduction: &Reduction, - input: &Rc, -) -> Option { - let sign_statistic = - estimates_raw_data_samples_of(input).map(|metric| raw_data_samples_sign_statistic(&metric)); - let stats = match reduction { - // The series count of the sources read bounds a PromQL fold's rows - // (series), not the rows of a SQL group. - Reduction::Reduce(_) if reads_table(input) => Some(PropagationStats { - row_count_statistic: Some(GROUP_ROW_COUNT), - ..PropagationStats::default() - }), - Reduction::Reduce(_) => Some(PropagationStats::default()), - Reduction::PerEntity => match input.non_asap() { - Some(NonASAPOp::PromqlSubquery { - range, - resolution: Some(resolution), - .. - }) if !resolution.is_zero() => Some(PropagationStats { - input_row_count: Some(range.as_nanos().div_ceil(resolution.as_nanos()) as u64), - ..PropagationStats::default() - }), - _ => None, - }, - }?; - Some(PropagationStats { - sign_statistic, - ..stats - }) -} - -/// The metric whose raw data samples, or plain sums of them, a composed node's -/// values are or estimate, or `None`: [`raw_data_samples_of_metric`]'s strict rule -/// over a composed candidate. An exact sum accumulator, a projection of -/// columns and a -/// heap estimate whose update weights carry the -/// [`WeightDomain::RawDataSamplesOfMetric`] requirement keep it, and so does a -/// quantile estimate of those values, built over the raw window or merged -/// from its tumbling windows: it is one of them, or between two of them, so it -/// shares their sign. Anything else (rate, arithmetic, other -/// functions) ends it. Only the sign of the values is read from it. -pub(crate) fn estimates_raw_data_samples_of(node: &Rc) -> Option { - match &node.operator { - Operator::NonASAP(NonASAPOp::TimeRange { child, .. }) => { - estimates_raw_data_samples_of(child) - } - Operator::NonASAP(NonASAPOp::TimeShift { child, shift }) if shift.at.is_none() => { - estimates_raw_data_samples_of(child) - } - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap()? { - ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), - child, - filter: None, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - }, - // Only a φ in [0, 1] picks one of the values; outside it the answer - // is -Inf or +Inf, whatever their sign. Merged tumbling windows of one - // definition are read through their `SummaryTumblingWindowChain`: - // a quantile of the merged state is still one of the tumbling windows' - // samples, or between two of them. - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: SketchStatistic::Quantile { q }, - }) if (0.0..=1.0).contains(q) => { - let build = match summary_input.summary_build()? { - chain @ ASAPOp::SummaryTumblingWindowChain { .. } => chain, - _ => summary_input.asap()?.clone(), - }; - match &build { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } - | ASAPOp::SummaryTumblingWindowChain { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - } - } - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - match summary_input.asap()? { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - weight_domain: WeightDomain::RawDataSamplesOfMetric { metric }, - .. - }, - filter: None, - .. - } => Some(metric.clone()), - _ => None, - } - } - // Renaming, reordering or dropping columns keeps every value. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - estimates_raw_data_samples_of(child) - } - _ => raw_data_samples_of_metric(node), - } -} - -/// Whether `node` reads a SQL table. -fn reads_table(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|node| { - matches!( - node.non_asap(), - Some(NonASAPOp::Scan { - source: Source::Table { .. }, - .. - }) - ) - }) -} - -/// Whether a grouping by `reduction` over an approximate `input` may decide -/// a row's group by an estimate: a grouping column other than a label (a -/// string) or the time index. Guarantees are per node, so any such column -/// may carry the estimate (column-level tracking is #689); a PromQL `by` -/// or `without` groups by labels only. `false` over exact input. -fn groups_by_estimate(reduction: &Reduction, input: &OperatorNode) -> bool { - let Some(by) = reduction.group_keys() else { - return false; - }; - if input - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - { - return false; - } - let schema = &input.schema; - let columns: Vec = if by.is_without() { - let value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); - (0..schema.fields.len()) - .filter(|i| !by.keys().contains(i) && Some(*i) != value) - .collect() - } else { - by.keys().to_vec() - }; - columns.into_iter().any(|i| { - Some(i) != schema.time_index - && !matches!( - schema.fields.get(i).map(|f| &f.dtype), - Some(FieldDataType::Plain(DataType::Utf8)) - ) - }) -} - -/// Whether `expr` reads any input column. -fn reads_columns(expr: &ScalarExpr) -> bool { - let mut reads = false; - expr.map_columns(&mut |column| { - reads = true; - column - }); - reads -} - -/// The built-in rule for `op`, with every statistic unknown (Stage 1 reads -/// none); `None` when no rule applies. -fn compose( - op: &CompositionOperator, - inputs: Vec, - local: Option<&AccuracyGuarantee>, -) -> Option { - propagate(op, &inputs, local, &PropagationStats::default()).ok() -} - -/// A projection that computes over its child's sample value: the guarantee -/// of the one item that reads it ([`project_column_guarantee`]). `None` when -/// no single item reads the value. -fn project_guarantee(cols: &[ProjectItem], child: &Rc) -> Option { - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; - let reads_value = |expr: &ScalarExpr| { - let mut reads = false; - expr.map_columns(&mut |column| { - reads |= column == value; - column - }); - reads - }; - let [item] = <[_; 1]>::try_from( - cols.iter() - .filter(|item| reads_value(&item.expr)) - .collect::>(), - ) - .ok()?; - scalar_guarantee(&item.expr, &|column| { - project_column_guarantee(child, column) - }) -} - -/// Whether `node` is a join, or reads one only through operators that keep -/// its columns (projections, filters, sorts, limits, deduplication). The -/// columns of such a node come from different inputs, so its sample value -/// says nothing about where the error lies. -fn derives_from_join(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Join { .. }) => true, - Some( - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } - | NonASAPOp::Dedup { child, .. }, - ) => derives_from_join(child), - _ => false, - } -} - -/// The guarantee of each output column of `node`, where it is known: every -/// column of an exact node is exact; a join beside exact inputs -/// ([`join_guarantee`]) gives the exact inputs' columns exact and the -/// approximate input's columns its guarantee; a projection of columns maps -/// its child's. `None` anywhere else, a computed projection included. -/// Guarantees are per node, so every column of an approximate input counts -/// as approximate; column-level tracking is #689. -/// -/// This assumes every approximate node of Stage 1 has an exact row set: -/// its guarantee bounds each row's values, never which rows exist (a -/// top-k estimate, whose rows are approximate, has no bound; -/// `estimate_has_bound`). A guarantee on row membership would need these -/// rules revisited. -fn column_guarantees(node: &OperatorNode) -> Option> { - let width = node.schema.fields.len(); - if let Some(guarantee) = node.guarantee.as_ref().filter(|g| g.is_exact()) { - return Some(vec![guarantee.clone(); width]); - } - match &node.operator { - Operator::NonASAP(NonASAPOp::Join { - kind: - JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - left, - right, - .. - }) => { - let input = join_input_guarantees(left, right)?; - (input.len() == width).then_some(input) - } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { - let child = column_guarantees(child)?; - cols.iter() - .map(|item| match item.expr { - ScalarExpr::Column(id) => child.get(id).cloned(), - _ => None, - }) - .collect() - } - _ => None, - } -} - -/// The guarantees of a join's input columns (`left ++ right`) when at most -/// one input is approximate and the others are exact: the exact input's -/// columns exact, the approximate input's columns its guarantee. -fn join_input_guarantees( - left: &OperatorNode, - right: &OperatorNode, -) -> Option> { - let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); - if !exact(left) && !exact(right) { - return None; - } - let side = |node: &OperatorNode| { - let guarantee = node.guarantee.clone()?; - Some(vec![guarantee; node.schema.fields.len()]) - }; - let mut columns = side(left)?; - columns.extend(side(right)?); - Some(columns) -} - -/// A join whose inputs are exact but one keeps that input's guarantee when -/// its predicate reads only exact columns (a cross join, or a key equality -/// on the exact inputs' columns): which rows match, and how often, then -/// does not depend on an estimate, and each output row carries one row of -/// the approximate input unchanged. Outer joins pad on the same exact -/// decision. A semi- or anti-join, a key on an approximate input's column -/// (per-node guarantees count all its columns approximate; #689), or two -/// approximate inputs have none. An exact join is composed before this. -/// Like [`column_guarantees`], this assumes the approximate input's row set -/// is exact: only its values carry error. -fn join_guarantee(node: &OperatorNode) -> Option { - let Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - pred, - left, - right, - }) = &node.operator - else { - return None; - }; - let columns = join_input_guarantees(left, right)?; - let pred_exact = - scalar_guarantee(&pred.0, &|id| columns.get(id).cloned()).is_some_and(|g| g.is_exact()); - if !pred_exact { - return None; - } - [left, right] - .into_iter() - .find_map(|input| input.guarantee.clone().filter(|g| !g.is_exact())) -} - -/// The guarantee of a projection over relational columns -/// ([`column_guarantees`]): each output column's as [`scalar_guarantee`] -/// (a `CASE` as [`case_guarantee`]), and the node's the weakest of them. -/// `None` when the child's column guarantees are unknown or any output -/// column has none. -fn relational_project_guarantee( - cols: &[ProjectItem], - child: &Rc, -) -> Option { - let columns = column_guarantees(child)?; - let column = |id: usize| columns.get(id).cloned(); - weakest( - cols.iter() - .map(|item| scalar_guarantee(&item.expr, &column)) - .collect::>>()?, - ) -} - -/// The weakest of `guarantees`: exact when all are, the approximate one -/// when there is one (or all approximate ones are equal), and otherwise -/// as an extremum of them ([`lipschitz_guarantee`]). -fn weakest(guarantees: Vec) -> Option { - let mut approximate: Vec = Vec::new(); - for guarantee in guarantees.into_iter().filter(|g| !g.is_exact()) { - if !approximate.contains(&guarantee) { - approximate.push(guarantee); - } - } - match approximate.len() { - 0 => Some(AccuracyGuarantee::exact("exact over exact operands")), - 1 => approximate.pop(), - _ => lipschitz_guarantee(approximate), - } -} - -/// A `CASE` whose operand and conditions are exact picks its branch by an -/// exact decision, so it keeps the weakest branch's guarantee (a literal or -/// NULL branch, or a missing `ELSE`, is exact). A condition over an -/// approximate value has none. -fn case_guarantee( - operand: Option<&ScalarExpr>, - branches: &[(ScalarExpr, ScalarExpr)], - else_expr: Option<&ScalarExpr>, - column: &dyn Fn(usize) -> Option, -) -> Option { - let exact = |expr: &ScalarExpr| scalar_guarantee(expr, column).is_some_and(|g| g.is_exact()); - if !operand.is_none_or(exact) || !branches.iter().all(|(when, _)| exact(when)) { - return None; - } - weakest( - branches - .iter() - .map(|(_, then)| then) - .chain(else_expr) - .map(|expr| scalar_guarantee(expr, column)) - .collect::>>()?, - ) -} - -/// The guarantee of `column` of a projection's `child`: over a join, the -/// column's own (`column_guarantees`); otherwise the child's sample -/// value has the child's as an operand ([`operand_guarantee`]) and other -/// columns (labels, timestamps) are exact. `None` when the child has no -/// sample value or the column's guarantee is unknown. -pub fn project_column_guarantee( - child: &Rc, - column: usize, -) -> Option { - // Over a join, each column's own guarantee ([`column_guarantees`]). - if derives_from_join(child) { - return column_guarantees(child)?.get(column).cloned(); - } - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; - if column == value { - operand_guarantee(child) - } else { - Some(AccuracyGuarantee::exact( - "column other than the sample value", - )) - } -} - -/// The guarantee of a scalar expression, given each column's: `scalar(v)` -/// and a SQL scalar subquery have their relation's, a literal or the -/// evaluation time is exact, arithmetic composes as -/// [`arithmetic_guarantee`], and `clamp`/`clamp_min`/`clamp_max`/`abs` as -/// [`lipschitz_guarantee`]. Any other expression (a comparison, a rounding, -/// a non-linear function) is exact over exact operands and has none -/// otherwise. -pub(crate) fn scalar_guarantee( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - let literal = |expr: &ScalarExpr| match expr { - ScalarExpr::Literal(ScalarValue::Float64(value)) => Some(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => Some(*value as f64), - _ => None, - }; - let operand = |expr: &ScalarExpr| (operand_of(expr, column), literal(expr)); - match expr { - ScalarExpr::Column(id) => column(*id), - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - node.guarantee.clone() - } - ScalarExpr::Literal(_) | ScalarExpr::CurrentTimestamp | ScalarExpr::EvalTimestamp => { - Some(AccuracyGuarantee::exact("constant")) - } - ScalarExpr::Negative { expr, .. } => { - lipschitz_guarantee(vec![scalar_guarantee(expr, column)?]) - } - ScalarExpr::Case { - operand, - branches, - else_expr, - } => case_guarantee(operand.as_deref(), branches, else_expr.as_deref(), column), - ScalarExpr::Arithmetic { - op, left, right, .. - } => arithmetic_guarantee( - op.clone(), - operand(left), - operand(right), - same_value(left, right), - ), - ScalarExpr::FunctionCall { name, args } if is_monotone_lipschitz(name) => { - lipschitz_guarantee( - args.iter() - .map(|arg| operand_of(arg, column)) - .collect::>()?, - ) - } - _ => { - let mut columns = Vec::new(); - expr.map_columns(&mut |id| { - columns.push(id); - id - }); - let exact = |g: Option| g.is_some_and(|g| g.is_exact()); - (columns.into_iter().all(|id| exact(column(id))) - && expr - .operator_refs() - .into_iter() - .all(|node| exact(node.guarantee.clone()))) - .then(|| AccuracyGuarantee::exact("exact over exact operands")) - } - } -} - -/// The guarantee of a scalar operand as an input to arithmetic or a -/// function: a planned operator's as [`operand_guarantee`], anything else -/// as `scalar_guarantee`. -pub fn operand_of( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - match expr { - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - operand_guarantee(node) - } - _ => scalar_guarantee(expr, column), - } -} - -/// `node`'s guarantee as an input to a computation over its values: a -/// Count-Min or Count Sketch bound restated as an absolute error over the -/// stream norm its summary's update weight allows (`stream_norm`, -/// `additive_as_absolute`). -pub fn operand_guarantee(node: &Rc) -> Option { - let guarantee = node.guarantee.clone()?; - Some(additive_as_absolute(guarantee, stream_norm(node).as_ref())) -} - -/// The stream norm that bounds the one Count-Min or Count Sketch summary -/// below `node`, by its update weight: the source's own norm when the -/// weight is the source's raw value column (the summary's input is the -/// source's scan, at most windowed or filtered); `|c|` times the source's -/// row count for a constant weight `c`. `None` for any other weight (a -/// computed value such as `m * 1000` or `rate(…)`), several summaries or -/// sources, or none. -pub(crate) fn stream_norm(node: &Rc) -> Option { - let summaries: Vec<_> = OperatorNode::reachable(node) - .into_iter() - .filter(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(algorithm), - .. - }) if matches!( - algorithm, - SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap - ) - ) - }) - .collect(); - let [summary] = summaries.as_slice() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - input: update, - child, - .. - }) = summary.asap() - else { - return None; - }; - let source = source(child)?; - match &update.weight { - SummaryInputExpr::Constant(weight) => Some(StreamNorm::ConstantWeight { - weight: *weight, - source, - }), - SummaryInputExpr::Column(column) if raw_value_column(child, column) => { - Some(StreamNorm::Source(source)) - } - _ => None, - } -} - -/// Whether `column` of `node` is its source's raw data sample value: `node` is -/// the source's scan, at most behind filters and windows that keep its -/// schema, and `column` resolves to the scan's sample value. -fn raw_value_column(node: &Rc, column: &ColumnRef) -> bool { - let Some(schema) = source_scan(node) else { - return false; - }; - let resolved = resolve_column_ref(column, schema).ok(); - resolved.is_some() && resolved == resolve_column_ref(&ColumnRef::SampleValue, schema).ok() -} - -/// The scan's schema when `node` is a source's scan, at most behind filters -/// and windows that keep its schema: its rows are the source's own. -fn source_scan(node: &Rc) -> Option<&Schema> { - let mut current = node; - loop { - if current.schema != node.schema { - return None; - } - match current.non_asap() { - Some(NonASAPOp::Filter { child, .. } | NonASAPOp::TimeRange { child, .. }) => { - current = child - } - Some(NonASAPOp::Scan { schema, .. }) => return Some(schema), - _ => return None, - } - } -} - -/// The one source (PromQL metric or SQL table) `node` reads; `None` when it -/// reads several or none. -fn source(node: &Rc) -> Option { - let sources: HashSet = OperatorNode::reachable(node) - .iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => Some(match source { - Source::TimeSeries { metric } => metric.clone(), - Source::Table { table_ref } => table_ref.clone(), - }), - _ => None, - }) - .collect(); - let mut sources = sources.into_iter(); - match (sources.next(), sources.next()) { - (Some(source), None) => Some(source), - _ => None, - } -} - -/// Whether two scalar operands are one value: the same column, or the same -/// planned operator read twice. -fn same_value(left: &ScalarExpr, right: &ScalarExpr) -> bool { - match (left, right) { - (ScalarExpr::Column(a), ScalarExpr::Column(b)) => a == b, - ( - ScalarExpr::PromqlScalarFromVector(a) | ScalarExpr::ScalarSubquery(a), - ScalarExpr::PromqlScalarFromVector(b) | ScalarExpr::ScalarSubquery(b), - ) => Rc::ptr_eq(a, b), - _ => false, - } -} - -/// Functions that are 1-Lipschitz and, for the relative-error rule, -/// monotone in each argument with `|f(x)|` scaling with `|x|`: `clamp` -/// (`min(max(x, lo), hi)`), `clamp_min`, `clamp_max` and `abs`. -fn is_monotone_lipschitz(name: &str) -> bool { - matches!( - name, - "promql_clamp" | "promql_clamp_min" | "promql_clamp_max" | "promql_abs" | "abs" - ) -} - -/// The guarantee of a 1-Lipschitz function of `args` (negation, `abs`, -/// `clamp`): its value is within the largest of its arguments' absolute -/// errors, by the union bound over them; likewise for relative errors, as -/// for an exact `max`/`min` (`ExactExtremum`, one value per argument). -/// Count-Min and Count Sketch bounds count as absolute errors. `None` for a -/// rank error or a mix of absolute and relative ones. -fn lipschitz_guarantee(args: Vec) -> Option { - let approximate: Vec<_> = args - .into_iter() - .filter(|g| !g.is_exact()) - .map(|g| additive_as_absolute(g, None)) - .collect(); - if approximate.is_empty() { - return Some(AccuracyGuarantee::exact( - "Lipschitz function of exact operands", - )); - } - propagate( - &CompositionOperator::ExactExtremum, - &approximate, - None, - &PropagationStats { - input_row_count: Some(1), - ..PropagationStats::default() - }, - ) - .ok() -} - -/// An arithmetic operand: its guarantee, and its value when it is a literal. -type Operand = (Option, Option); - -/// The guarantee of `left op right` (#669, MVP). Count-Min and Count Sketch -/// bounds count as absolute errors (normalized bound times stream norm). -/// - `x ± y`: absolute errors add (union bound on the failure -/// probabilities); `x ± c` (`c` exact) keeps an absolute error. -/// - `c · x`, `x / c` (`c` a nonzero literal): an absolute error scales by -/// `|c|` (or `1/|c|`); a relative error is kept, as it is by `x · y` or -/// `x / y` with `y` exact. -/// - `x · y` of two relative errors: relative `a + b + a·b`. -/// - `x / y` with `y` a relative error and `x` a relative error or exact: -/// relative `(a + b)/(1 − b)` (`a = 0` for an exact `x`), none when -/// `b ≥ 1`. -/// - `x · y` of two other estimates, `x / y` with any other approximate -/// `y`: a bound that needs enforced operand domains, left symbolic -/// ([`operand_domain_statistic`]) for plan selection to bind. -/// - Everything else (a relative error shifted by a constant or added to -/// another, a rank error, `%`, `^`, `atan2`) has none. -/// -/// `same_operand`: both sides are one estimate (the same planned operator), -/// which fails or holds as one event, so its failure probability counts -/// once. Two estimates count once each, even over one shared summary: the -/// built-in guarantees hold per estimate, not simultaneously. -fn arithmetic_guarantee( - op: ArithmeticOpKind, - left: Operand, - right: Operand, - same_operand: bool, -) -> Option { - let (l, r) = ( - additive_as_absolute(left.0?, None), - additive_as_absolute(right.0?, None), - ); - let once = same_operand.then(|| l.spec.failure_probability.clone()); - let statistic = operand_domain_statistic(&op, &l, &r); - let composed = match (op, l.is_exact(), r.is_exact()) { - (_, true, true) => return Some(AccuracyGuarantee::exact("arithmetic over exact operands")), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, true, false) => shifted(r), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, false, true) => shifted(l), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, false, false) => compose( - &CompositionOperator::Lipschitz { constant: 1.0 }, - vec![l], - Some(&r), - ), - (ArithmeticOpKind::Mul, true, false) => scaled(r, left.1.map(f64::abs)), - (ArithmeticOpKind::Mul, false, true) => scaled(l, right.1.map(f64::abs)), - (ArithmeticOpKind::Mul, false, false) if statistic.is_none() => relative_product(l, r), - (ArithmeticOpKind::Div, false, true) => match right.1 { - Some(0.0) => None, - c => scaled(l, c.map(|c| 1.0 / c.abs())), - }, - (ArithmeticOpKind::Div, _, false) if statistic.is_none() => propagate( - &CompositionOperator::ExactDivision, - &[l, r], - None, - &PropagationStats::default(), - ) - .ok(), - (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, _, _) => symbolic(l, r, statistic?), - _ => None, - }?; - let mut composed = composed; - if let Some(failure_probability) = once { - composed.spec.failure_probability = failure_probability; - } - Some(composed) -} - -/// The statistic a product or quotient of `l` and `r` leaves symbolic on -/// its operands' domains, when it is one whose bound needs them: a product -/// of two estimates, other than of two relative errors -/// ([`PRODUCT_OPERAND_DOMAINS`]); a quotient with an approximate -/// denominator, other than of relative or exact over relative -/// (`QUOTIENT_OPERAND_DOMAINS`). `None` for any other operation (a -/// product or quotient of relative errors has a closed form), or when an -/// operand's error has no value form (a rank error). Plan selection -/// (Stage 3) finds the operations to bind with this same rule. -pub fn operand_domain_statistic( - op: &ArithmeticOpKind, - l: &AccuracyGuarantee, - r: &AccuracyGuarantee, -) -> Option<&'static str> { - let valued = |g: &AccuracyGuarantee| { - g.is_exact() - || matches!( - g.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE | AccuracyMetric::Relative - ) - }; - let relative = - |g: &AccuracyGuarantee| g.is_exact() || g.spec.accuracy_metric == AccuracyMetric::Relative; - if !(valued(l) && valued(r)) { - return None; - } - match op { - ArithmeticOpKind::Mul - if !l.is_exact() - && !r.is_exact() - && !(l.spec.accuracy_metric == AccuracyMetric::Relative - && r.spec.accuracy_metric == AccuracyMetric::Relative) => - { - Some(PRODUCT_OPERAND_DOMAINS) - } - ArithmeticOpKind::Div if !r.is_exact() && !(relative(l) && relative(r)) => { - Some(QUOTIENT_OPERAND_DOMAINS) - } - _ => None, - } -} - -/// A product of two relative-error estimates: within relative -/// `(1 + a)(1 + b) − 1 = a + b + a·b`, whatever the signs. `None` for any -/// other pair. -fn relative_product(l: AccuracyGuarantee, r: AccuracyGuarantee) -> Option { - if l.spec.accuracy_metric != AccuracyMetric::Relative - || r.spec.accuracy_metric != AccuracyMetric::Relative - { - return None; - } - let bound = BoundExpr::Sum { - terms: vec![ - l.spec.error.clone(), - r.spec.error.clone(), - BoundExpr::Product { - factors: vec![l.spec.error.clone(), r.spec.error.clone()], - }, - ], - }; - Some(binary_guarantee( - AccuracyMetric::Relative, - bound, - l, - r, - None, - CompositionOperator::ExactProduct, - "relative_product_union_bound", - )) -} - -/// `x` times an exact factor (`Some(|c|)` when it is a literal): a -/// relative error is kept; an absolute error scales by `|c|`. -fn scaled(input: AccuracyGuarantee, factor: Option) -> Option { - match (input.spec.accuracy_metric, factor) { - (AccuracyMetric::Relative, _) => Some(input), - (_, Some(constant)) => compose( - &CompositionOperator::Lipschitz { constant }, - vec![input], - None, - ), - _ => None, - } -} - -/// `x` plus an exact value: an absolute error is kept. -fn shifted(input: AccuracyGuarantee) -> Option { - compose( - &CompositionOperator::Lipschitz { constant: 1.0 }, - vec![input], - None, - ) -} - -/// A product or quotient whose bound needs its operands' domains, which -/// Stage 1 does not read: the bound is the symbolic `statistic` -/// ([`operand_domain_statistic`]), absolute, with the union bound on the -/// failure probability. -fn symbolic( - l: AccuracyGuarantee, - r: AccuracyGuarantee, - statistic: &'static str, -) -> Option { - let (operator, rule) = match statistic { - QUOTIENT_OPERAND_DOMAINS => ( - CompositionOperator::ExactDivision, - "absolute_division_needs_operand_domains", - ), - _ => ( - CompositionOperator::ExactProduct, - "absolute_product_needs_operand_domains", - ), - }; - Some(binary_guarantee( - AccuracyMetric::ABSOLUTE, - BoundExpr::Unknown { - statistic: statistic.into(), - }, - l, - r, - Some(statistic), - operator, - rule, - )) -} - -/// A guarantee over two operands with the union bound on their failure -/// probabilities, and its provenance. -fn binary_guarantee( - metric: AccuracyMetric, - bound: BoundExpr, - l: AccuracyGuarantee, - r: AccuracyGuarantee, - unavailable: Option<&str>, - operator: CompositionOperator, - rule: &str, -) -> AccuracyGuarantee { - let failure_probability = ProbabilityExpr::UnionBound { - terms: vec![ - l.spec.failure_probability.clone(), - r.spec.failure_probability.clone(), - ], - }; - let provenance = [l, r] - .into_iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g), - }) - .chain( - unavailable.map(|statistic| GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }), - ) - .chain([GuaranteeSource::CompositionStep { - operator, - rule: rule.into(), - }]) - .collect(); - AccuracyGuarantee::new(metric, bound, failure_probability, provenance) -} - -/// Whether the default accuracy model's bound for `family` answers `query` -/// over `update`. A heap sketch's bound is per item frequency, not Top-K -/// membership; a Count-Min bound assumes non-negative weights. -pub(crate) fn estimate_has_bound( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> bool { - let count_min = matches!( - family, - FieldDataType::Sketch(algorithm) - if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) - ); - !matches!(query, SketchStatistic::TopK { .. }) - && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) -} - -/// DDS quantiles use linear interpolation of neighbouring representatives. -/// Relative error survives that convex combination only without cancellation. -/// Endpoints select an extremum; interior quantiles need a sign proof. Keep a -/// raw metric's requirement symbolic so Stage 3 can bind declared source facts. -fn interpolation_guarantee( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, - input: &Rc, - mut guarantee: AccuracyGuarantee, -) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch)) - && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); - if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { - return Some(guarantee); - } - let metric = match &update.weight_domain { - WeightDomain::RawDataSamplesOfMetric { metric } => metric.clone(), - _ if matches!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ) => - { - estimates_raw_data_samples_of(input)? - } - _ => return None, - }; - guarantee.spec.error = asap_types::accuracy::BoundExpr::Product { - factors: vec![ - guarantee.spec.error, - asap_types::accuracy::BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic(&metric), - }, - ], - }; - Some(guarantee) -} - -/// A Count-Min estimate whose weights carry the open requirement that they -/// are raw data samples of one metric ([`WeightDomain::RawDataSamplesOfMetric`]): its local -/// guarantee with the bound gated on that metric's sign -/// ([`raw_data_samples_sign_statistic`]), which plan selection binds to 1 when the -/// metric is declared a counter. `None` for any other estimate. -pub(crate) fn estimate_bound_over_raw_data_samples( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> Option { - let WeightDomain::RawDataSamplesOfMetric { metric } = &update.weight_domain else { - return None; - }; - let proven = SummaryUpdate { - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples, - }, - ..update.clone() - }; - if !estimate_has_bound(family, &proven, query) { - return None; - } - let _ = metric; - todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters, its bound gated on the metric's sign") -} - -pub fn split_and(expr: &ScalarExpr, out: &mut Vec) { - match expr { - ScalarExpr::BoolAnd(items) => items.iter().for_each(|item| split_and(item, out)), - other => out.push(other.clone()), - } -} - -/// The columns `pred` pins to one literal value each: its conjuncts -/// `c = v` and `c IN (v)`. -fn pinned_columns(pred: &ScalarExpr) -> Vec { - let mut conjuncts = Vec::new(); - split_and(pred, &mut conjuncts); - conjuncts.iter().filter_map(pinned_column).collect() -} - -/// The column `conjunct` pins to one literal value (`c = v`, `c IN (v)`). -pub fn pinned_column(conjunct: &ScalarExpr) -> Option { - match conjunct { - ScalarExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - .. - } => match (&**left, &**right) { - (ScalarExpr::Column(column), ScalarExpr::Literal(_)) - | (ScalarExpr::Literal(_), ScalarExpr::Column(column)) => Some(*column), - _ => None, - }, - ScalarExpr::InList { - expr, - list, - negated: false, - } => match (&**expr, list.as_slice()) { - (ScalarExpr::Column(column), [ScalarExpr::Literal(_)]) => Some(*column), - _ => None, - }, - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::summary_type::{NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr}; - - /// A Count-Min estimate has a bound only over weights proven - /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. - #[test] - fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let summary_type = FieldDataType::Sketch(SketchAlgorithm::Cms); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - let with = |weight_domain| SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain, - }; - assert!(!estimate_has_bound( - &summary_type, - &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), - &query - )); - assert!(!estimate_has_bound( - &summary_type, - &with(WeightDomain::UnknownOrSigned), - &query - )); - assert!(estimate_has_bound( - &summary_type, - &with(WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }), - &query - )); - } -} diff --git a/crates/accuracy/src/propagation_stats.rs b/crates/accuracy/src/propagation_stats.rs deleted file mode 100644 index 58db409b9..000000000 --- a/crates/accuracy/src/propagation_stats.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Scoped source contracts and statistics the accuracy rules consult. -use asap_types::accuracy::GuaranteeSource; - -/// Statistics a propagation rule may consult. Every field is optional and -/// defaults to "unknown": a rule that needs a missing statistic emits a -/// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct PropagationStats { - /// Provenance for supplied evidence (source, observation identity, etc.). - pub(crate) evidence_provenance: Vec, - /// Whether every input value is known to be non-negative — required by - /// the multiplicative relative-error rule, which is unsound across a - /// sign change. - pub(crate) values_non_negative: Option, - /// The statistic an unknown sign is named by when the folded values are - /// one metric's raw data samples or plain sums of them - /// ([`raw_data_samples_sign_statistic`](crate::composition::raw_data_samples_sign_statistic)): - /// the relative bound is then gated on it, and plan selection binds it - /// to 1 when the metric is declared a counter. `None`: the bare - /// `values_non_negative`, which nothing binds. - pub(crate) sign_statistic: Option, - /// Number of input rows an exact aggregation consumes (e.g. the number - /// of groups a function folds), for exact aggregate union bounds - /// bound over per-input failures. - pub(crate) input_row_count: Option, - /// The statistic an unknown row count is named by (default - /// `input_row_count`), so a fold whose rows the data workload's input - /// cardinality does not bound can leave a statistic nothing binds. - pub(crate) row_count_statistic: Option<&'static str>, - /// Lower confidence bound of the kth selected TopK item, after widening - /// the interval by the sketch's own estimation error. - pub(crate) topk_selected_lower_bound: Option, - /// Greatest upper confidence bound among excluded TopK items, after - /// widening the interval by the sketch's own estimation error. - pub(crate) topk_excluded_upper_bound: Option, - /// Union-bound failure probability of all intervals used by the margin - /// certificate. - pub(crate) topk_interval_failure_probability: Option, -} diff --git a/crates/accuracy/src/readout/mod.rs b/crates/accuracy/src/readout/mod.rs new file mode 100644 index 000000000..fc4c9b8bc --- /dev/null +++ b/crates/accuracy/src/readout/mod.rs @@ -0,0 +1,65 @@ +//! The metric of a readout's error: what a `SummaryEstimate` of a sketch +//! state may be off in. + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroup, Memo}; +use asap_types::summary_type::{parameter_list, SketchStatistic}; + +use crate::guarantee::guarantee_before_failure_probability_check; + +/// The metric of the error reading `statistic` from a state of +/// `summary_type` has: its first listed θ's guarantee's, whatever that +/// guarantee's failure probability (HLL's distinct count reads a relative +/// error although its failure probability is unknown). `None` for an exact +/// state and for a summary type with no listed θ or no guarantee for the +/// statistic (UnivMon's distinct count). +pub(crate) fn readout_error_metric( + summary_type: &FieldDataType, + statistic: &SketchStatistic, +) -> Option { + let FieldDataType::Sketch(algorithm) = summary_type else { + return None; + }; + let parameters = parameter_list(*algorithm).first()?; + Some( + guarantee_before_failure_probability_check(summary_type, parameters, statistic, 1.0)? + .spec + .accuracy_metric, + ) +} + +/// The metric of `candidate`'s error when it is a readout +/// (`SummaryEstimate`) of a sketch state ([`readout_error_metric`]); `None` +/// for any other candidate. +pub(crate) fn sketch_readout_error_metric( + memo: &Memo, + candidate: &CandidateForMemoGroup, +) -> Option { + let (summary_type, statistic) = sketch_readout(memo, candidate)?; + readout_error_metric(&summary_type, statistic) +} + +/// The summary type of the sketch state `candidate` reads and the statistic +/// it reads from it, when it is a readout (`SummaryEstimate`) of a sketch +/// state (`GroupProperties::sketch_algorithm_of_state`). +pub(crate) fn sketch_readout<'a>( + memo: &Memo, + candidate: &'a CandidateForMemoGroup, +) -> Option<(FieldDataType, &'a SketchStatistic)> { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query, + }) = &candidate.expression.operator + else { + return None; + }; + let algorithm = memo.groups()[summary_input.0 as usize] + .group_properties + .sketch_algorithm_of_state()?; + Some((FieldDataType::Sketch(algorithm), query)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/readout/tests.rs b/crates/accuracy/src/readout/tests.rs new file mode 100644 index 000000000..b8a0418b5 --- /dev/null +++ b/crates/accuracy/src/readout/tests.rs @@ -0,0 +1,28 @@ +use super::*; +use asap_types::summary_type::{ExactKind, SketchAlgorithm}; + +/// A readout's metric is its algorithm's: KLL quantiles read a rank +/// error, DDSketch quantiles and HLL distinct counts a relative one +/// (HLL's although its failure probability is unknown); an exact state +/// reads exactly. +#[test] +fn a_readouts_metric_is_its_algorithms() { + let quantile = SketchStatistic::Quantile { q: 0.9 }; + let sketch = |algorithm| FieldDataType::Sketch(algorithm); + assert_eq!( + readout_error_metric(&sketch(SketchAlgorithm::Kll), &quantile), + Some(AccuracyMetric::Rank) + ); + assert_eq!( + readout_error_metric(&sketch(SketchAlgorithm::DDSketch), &quantile), + Some(AccuracyMetric::Relative) + ); + assert_eq!( + readout_error_metric(&sketch(SketchAlgorithm::Hll), &SketchStatistic::Cardinality), + Some(AccuracyMetric::Relative) + ); + assert_eq!( + readout_error_metric(&FieldDataType::exact(ExactKind::Sum), &quantile), + None + ); +} diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs new file mode 100644 index 000000000..6c8471430 --- /dev/null +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -0,0 +1,336 @@ +//! Readout requirements (concepts doc, section 4, "Readout requirements" +//! and "Validity caps and pruning caps"): the caps the queries that reach +//! a readout pass down through their consumers' error contracts, and the +//! required-accuracy levels each readout gets from them. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::Operator; +use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId, RequiredAccuracy}; + +use crate::metric_rule::{query_roots_with_target, ErrorContractPerMetric}; +use crate::ErrorMetricsOfMemo; + +/// The error grid of the readouts of a query that combines several +/// ([`query_levels`]), loosest first. +const MAX_ERROR_LEVELS: [f64; 7] = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; + +/// The failure-probability grid of the readouts of a composed query: the +/// powers of ten from 1e-2 down to 1e-5, continued down to the first one at +/// or below a tenth of the query's per-value failure probability, so that up +/// to ten readouts each find a level and the levels sum within it (δ = 0.01 +/// over 10 000 series is 1e-6 per value: two quantiles of a ratio read at +/// 1e-7 each). +fn failure_probability_levels(per_value_failure_probability: f64) -> Vec { + let lowest = (per_value_failure_probability / 10.0).min(1e-5); + let deepest_exponent = (-lowest.log10() - 1e-9).ceil().min(300.0) as i32; + (2..=deepest_exponent) + .map(|exponent| 1.0 / 10f64.powi(exponent)) + .collect() +} + +/// Per group, by group id: `None` when no query with an accuracy target +/// reaches it; else per metric its output may be off in, per query, the +/// largest error in that metric the group's output may have for the query, +/// $\max_q \varepsilon_q / \lambda_{q,c,i}$ capped by the validity caps on +/// the way. A metric with no query has no cap: it reaches the group only +/// through inputs the output does not depend on ($\beta = 0$, a count's). +pub(crate) type Caps = Vec>>>; + +/// The required-accuracy levels one sketch readout gets +/// ([`readout_levels`]), with the queries reaching it whose failure +/// probability has no per-value one there. +#[derive(Debug, Clone, PartialEq)] +pub struct ReadoutRequiredAccuracy { + /// One per readout candidate Pass 2 keeps: Pass 1's takes the first, + /// a copy each further one. + pub levels: Vec, + /// The queries, by index, whose target sets a failure probability + /// without a per-value one at the readout (the values it covers or the + /// readout group's values per evaluation unknown). + pub queries_without_per_value_failure_probability: Vec, +} + +/// The required-accuracy levels of the sketch readouts without a +/// requirement yet, per readout ([`ReadoutRequiredAccuracy`]), from +/// `error_metrics` of `memo` ([`ErrorMetricsOfMemo::of`]). `order` is the +/// memo's topological order (`Memo::topological_order`). +/// - A query whose path reads one readout group gives one level: its target +/// divided by the path's factor, the readout group's cap, at the target's +/// failure probability per readout value ([`query_levels`]). +/// - A composed query (several readout groups, such as a ratio) gives every +/// level of the grid [`MAX_ERROR_LEVELS`] × [`failure_probability_levels`] +/// not looser than the cap and the per-value failure probability (the cap +/// itself when no error level is), one δ level of 1 when the target sets +/// no failure probability. +/// - A readout group reached by several queries gets the union of their +/// levels, in query order; one no query with a target reaches in the +/// readout's metric gets none. Readouts of one group in one metric get +/// the same levels. +/// +/// A query whose target sets a failure probability that has no per-value +/// one at the readout gives no level and is listed instead: Pass 2 prunes +/// a readout no query gives a level +/// (`RootPruneReason::ValuesPerEvaluationUnknown`), and lets a listed query +/// read the group only exactly when another query gives it a level. +pub fn readout_levels( + memo: &Memo, + error_metrics: &ErrorMetricsOfMemo, + order: &[MemoGroupId], +) -> BTreeMap { + let caps = caps(memo, &error_metrics.error_contract, order); + let mut readouts: Vec<(CandidateForMemoGroupId, AccuracyMetric)> = Vec::new(); + for (index, group) in memo.groups().iter().enumerate() { + for (index_in_group, candidate) in group.candidates.iter().enumerate() { + let readout = CandidateForMemoGroupId { + group: MemoGroupId(index as u32), + index_in_group, + }; + if candidate + .input_requirements + .first() + .is_some_and(|input| input.required_accuracy.is_none()) + { + if let Some(metric) = error_metrics.readout_error_metric(readout) { + readouts.push((readout, metric)); + } + } + } + } + let readout_groups: BTreeSet = + readouts.iter().map(|(readout, _)| readout.group).collect(); + let composed = composed_queries(memo, order, &readout_groups); + let mut levels_of_group: HashMap<(MemoGroupId, AccuracyMetric), ReadoutRequiredAccuracy> = + HashMap::new(); + readouts + .into_iter() + .map(|(readout, metric)| { + let group = readout.group; + let levels = levels_of_group + .entry((group, metric)) + .or_insert_with(|| group_levels(memo, &caps, &composed, group, metric)) + .clone(); + (readout, levels) + }) + .collect() +} + +/// The levels of readout group `group` in `metric` ([`readout_levels`]). +fn group_levels( + memo: &Memo, + caps: &Caps, + composed: &BTreeSet, + group: MemoGroupId, + metric: AccuracyMetric, +) -> ReadoutRequiredAccuracy { + let readout_values = memo.max_values_per_evaluation(group); + let mut levels: Vec = Vec::new(); + let mut queries_without_per_value_failure_probability = Vec::new(); + let group_caps = caps[group.0 as usize] + .as_ref() + .and_then(|caps| caps.get(&metric)) + .into_iter() + .flatten(); + for (&query, &cap) in group_caps { + let Some(target) = &memo.queries()[query].accuracy_target else { + continue; + }; + let Some(query_levels) = query_levels( + target, + memo.queries()[query].number_of_values_per_query_evaluation, + readout_values, + metric, + cap, + composed.contains(&query), + ) else { + queries_without_per_value_failure_probability.push(query); + continue; + }; + for level in query_levels { + if !levels.contains(&level) { + levels.push(level); + } + } + } + ReadoutRequiredAccuracy { + levels, + queries_without_per_value_failure_probability, + } +} + +/// The requirement levels one query gives a readout in `metric` whose cap +/// for it is `cap`; `None` when its target's failure probability has no +/// per-value one. A whole-result δ splits over the values one evaluation +/// returns, $n_q$; a reduction (`max`, `min`, `avg`) fails when any +/// readout it reads does, so δ also splits over the readout group's values +/// per evaluation, `readout_values` (`Memo::max_values_per_evaluation`): +/// the per-value failure probability is +/// $\delta / \max(n_q, \text{readout values})$ (`max(quantile_over_time(0.9, +/// m[5m]))` over 1000 series at δ = 0.01: 1e-5; `SELECT count(DISTINCT +/// dst)`, one readout value: 0.01), and needs both known when the target +/// sets a δ. +fn query_levels( + target: &AccuracyTarget, + number_of_values_per_query_evaluation: Option, + readout_values: Option, + metric: AccuracyMetric, + cap: f64, + composed: bool, +) -> Option> { + let per_value_failure_probability = match target.failure_probability { + None => 1.0, + Some(delta) => { + let values_covered = target + .accuracy_scope + .max_values_covered(number_of_values_per_query_evaluation)?; + delta / values_covered.max(readout_values?).max(1) as f64 + } + }; + let level = |max_error, failure_probability| RequiredAccuracy { + metric, + max_error, + failure_probability, + }; + if !composed { + return Some(vec![level(cap, per_value_failure_probability)]); + } + let within = |grid: &[f64], limit: f64| -> Vec { + let kept: Vec = grid.iter().copied().filter(|&x| x <= limit).collect(); + if kept.is_empty() { + vec![limit] + } else { + kept + } + }; + let errors = within(&MAX_ERROR_LEVELS, cap); + let failure_probabilities = if target.failure_probability.is_none() { + vec![1.0] + } else { + within( + &failure_probability_levels(per_value_failure_probability), + per_value_failure_probability, + ) + }; + Some( + errors + .iter() + .flat_map(|&max_error| { + failure_probabilities + .iter() + .map(move |&failure_probability| level(max_error, failure_probability)) + }) + .collect(), + ) +} + +/// Each group's caps ([`Caps`]), one walk down from the query roots: each +/// query root group with a target is reached, with the target in each of +/// the metrics it allows ([`query_roots_with_target`]); then, consumers +/// before producers, every candidate of a reached group reaches its inputs +/// and passes them each metric its error contract (`error_contract`, per +/// group and candidate) has, with the caps +/// $\min(b_{c,i},\ \text{cap}/\beta_{c,i})$, keeping the largest per query +/// and metric (the bottleneck over paths); an input with $\beta_{c,i} = 0$ +/// gets the metric but no cap through that candidate. +pub(crate) fn caps( + memo: &Memo, + error_contract: &[Vec], + order: &[MemoGroupId], +) -> Caps { + let mut caps: Caps = vec![None; memo.groups().len()]; + for (query, root, target, metrics) in query_roots_with_target(memo) { + let root_caps = caps[root.0 as usize].get_or_insert_with(HashMap::new); + for metric in metrics { + raise(root_caps, metric, query, target.max_error); + } + } + for &group in order.iter().rev() { + let Some(group_caps) = caps[group.0 as usize].clone() else { + continue; + }; + let candidates = &memo.groups()[group.0 as usize].candidates; + for (consumer, contract_per_metric) in + candidates.iter().zip(&error_contract[group.0 as usize]) + { + let children = consumer.expression.operator.children(); + for &child in &children { + caps[child.0 as usize].get_or_insert_with(HashMap::new); + } + for (metric, contract) in AccuracyMetric::ALL.into_iter().zip(contract_per_metric) { + let (Some(caps_per_query), Some(contract)) = (group_caps.get(&metric), contract) + else { + continue; + }; + for (input, &child) in children.iter().enumerate() { + let child_caps = caps[child.0 as usize].get_or_insert_with(HashMap::new); + child_caps.entry(metric).or_default(); + let factor = contract.error_factor_per_input[input]; + // An input the output does not depend on (a count's, + // β = 0) gets no cap from it. + if factor > 0.0 { + let holds_up_to = contract.linear_bound_holds_up_to_per_input[input] + .unwrap_or(f64::INFINITY); + for (&query, &cap) in caps_per_query { + raise(child_caps, metric, query, holds_up_to.min(cap / factor)); + } + } + } + } + } + } + caps +} + +/// The queries that combine several readouts: a candidate they reach has +/// two or more inputs below which a readout group lies (a ratio of two +/// quantiles). Readouts of one value by different summary types are +/// alternatives in one group's candidates, not a combination. +fn composed_queries( + memo: &Memo, + order: &[MemoGroupId], + readout_groups: &BTreeSet, +) -> BTreeSet { + let mut reads_a_readout = vec![false; memo.groups().len()]; + let mut composed = BTreeSet::new(); + for &group in order { + let candidates = &memo.groups()[group.0 as usize].candidates; + let inputs_reading = |operator: &Operator| { + operator + .children() + .into_iter() + .filter(|child| reads_a_readout[child.0 as usize]) + .count() + }; + let most_inputs_reading = candidates + .iter() + .map(|candidate| inputs_reading(&candidate.expression.operator)) + .max() + .unwrap_or(0); + if most_inputs_reading > 1 { + composed.extend( + memo.groups()[group.0 as usize] + .reached_by_queries() + .iter() + .copied(), + ); + } + reads_a_readout[group.0 as usize] = + readout_groups.contains(&group) || most_inputs_reading > 0; + } + composed +} + +/// Keeps the larger of `cap` and the known cap of `query` in `metric`. +fn raise( + caps: &mut HashMap>, + metric: AccuracyMetric, + query: usize, + cap: f64, +) { + let known = caps.entry(metric).or_default().entry(query).or_insert(cap); + *known = known.max(cap); +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/readout_caps/tests.rs b/crates/accuracy/src/readout_caps/tests.rs new file mode 100644 index 000000000..cf1acf089 --- /dev/null +++ b/crates/accuracy/src/readout_caps/tests.rs @@ -0,0 +1,156 @@ +use super::*; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::QueryRoot; +use asap_types::stages::MemoQuery; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; +use asap_types::test_support::memo_builders::{ + estimate, group, own_candidate, quantile_over_time, query, scan_group, sketch_state, + with_series, +}; +use asap_types::test_support::memo_fixtures::{aggregate, arithmetic}; + +/// The levels [`readout_levels`] gives the readout candidate `readout`. +fn levels_of(memo: &Memo, readout: MemoGroupId) -> Vec { + let order = memo.topological_order().expect("acyclic"); + readout_levels(memo, &ErrorMetricsOfMemo::of(memo, &order), &order) + .remove(&own_candidate(readout)) + .expect("a sketch readout") + .levels +} + +/// A whole-result failure probability is split over the values one +/// evaluation returns: δ = 0.01 over 100 series is 1e-4 per value. +#[test] +fn a_whole_result_failure_probability_is_split_per_value() { + let mut memo = Memo::new(&with_series("m", 100)); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(100), + }); + let [required] = levels_of(&memo, readout)[..] else { + panic!("one requirement"); + }; + assert!((required.failure_probability - 1e-4).abs() < 1e-12); +} + +/// The snapshot workload +/// `ratio-of-quantiles-1h-mvp`: `quantile_over_time(0.9, data[1h]) / +/// quantile_over_time(0.5, data[1h])` with relative error 0.05 and +/// δ = 0.01 over the whole result of 10 000 series is δ = 1e-6 per +/// value, below the grid's 1e-5. The δ levels continue in powers of ten +/// below it, so both readouts find levels whose sum stays within 1e-6 +/// and (K) can hold. +#[test] +fn a_ratio_whose_per_value_failure_probability_is_below_the_grid_splits_it_over_its_readouts() { + let mut memo = Memo::new(&with_series("m", 10_000)); + let (_, state, p90) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 3600); + let p50 = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.5 }); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p90, p50)); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(ratio), + accuracy_target: Some( + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ), + number_of_values_per_query_evaluation: Some(10_000), + }); + let smallest_failure_probability = |readout| { + let levels = levels_of(&memo, readout); + assert!(!levels.is_empty(), "every readout has a level"); + levels + .into_iter() + .map(|required| required.failure_probability) + .fold(f64::INFINITY, f64::min) + }; + let (p90_delta, p50_delta) = ( + smallest_failure_probability(p90), + smallest_failure_probability(p50), + ); + assert!( + p90_delta + p50_delta <= 1e-6, + "{p90_delta} + {p50_delta} > 1e-6" + ); +} + +/// A `max` reads every readout of one evaluation, and fails when any of +/// them does: `max(quantile_over_time(0.9, m[5m]))` over 1000 series +/// with δ = 0.01 returns one value, yet each of its 1000 readouts must +/// fail at most 1e-5 for the union bound to hold. +#[test] +fn a_max_over_many_readouts_splits_the_failure_probability_over_them() { + let mut memo = Memo::new(&with_series("m", 1000)); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + let max = group( + &mut memo, + aggregate(Reduction::by(vec![]), AggIntent::Max { col: None }, readout), + ); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(max), + accuracy_target: Some( + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ), + number_of_values_per_query_evaluation: Some(1), + }); + let levels = levels_of(&memo, readout); + assert!(!levels.is_empty()); + for level in levels { + assert!( + level.failure_probability <= 1e-5 * (1.0 + 1e-9), + "{level:?}" + ); + } +} + +/// A count reads no value (β = 0), so the readouts below it get no +/// level from it: `count(quantile_over_time(0.9, m[5m]))` requires +/// nothing of its readout, not an infinite error. +#[test] +fn a_count_over_readouts_gives_them_no_level() { + let mut memo = Memo::new(&with_series("m", 1000)); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + let count = group( + &mut memo, + aggregate(Reduction::by(vec![]), AggIntent::Count, readout), + ); + query( + &mut memo, + count, + Some(AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative)), + ); + assert_eq!(levels_of(&memo, readout), vec![]); +} + +/// A readout of one value per evaluation splits no failure probability +/// over its scanned series: `SELECT count(DISTINCT dst) FROM flows` at +/// δ = 0.01 over 1000 groups of rows reads one distinct count, so its +/// KMV readout gets 0.01, not 1e-5. +#[test] +fn a_readout_of_one_value_keeps_the_whole_failure_probability() { + let mut memo = Memo::new(&with_series("flows", 1000)); + let rows = scan_group(&mut memo, "flows", &["dst"], vec![]); + let state = sketch_state(&mut memo, rows, SketchAlgorithm::Kmv, Reduction::by(vec![])); + let distinct = estimate(&mut memo, state, SketchStatistic::Cardinality); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(distinct), + accuracy_target: Some( + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ), + number_of_values_per_query_evaluation: Some(1), + }); + let [required] = levels_of(&memo, distinct)[..] else { + panic!("one requirement"); + }; + assert!( + (required.failure_probability - 0.01).abs() < 1e-12, + "{required:?}" + ); +} diff --git a/crates/accuracy/src/requirements.rs b/crates/accuracy/src/requirements.rs deleted file mode 100644 index 58e7d78d6..000000000 --- a/crates/accuracy/src/requirements.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Whether a producer meets a consumer's logical requirements: -//! $\alpha^{\log}$ of a logical edge, and the limit an evaluated bound is -//! compared against. - -#[cfg(test)] -use asap_types::accuracy::AccuracyGuarantee; -use asap_types::stages::{CandidateForMemoGroupId, Memo}; - -use crate::AccuracyEvidence; - -/// $\alpha^{\log}$ of one edge (concepts doc, section 2): for a readout's -/// input, the producer's guarantee for the readout's statistic at the -/// required failure probability is within the required error, and a -/// top-k's heap holds k; for any other input, the metric rule (section 4). -pub fn producer_satisfies_logical_requirements( - memo: &Memo, - producer: CandidateForMemoGroupId, - consumer: CandidateForMemoGroupId, - input_index_of_consumer: usize, - evidence: &dyn AccuracyEvidence, -) -> bool { - let _ = (memo, producer, consumer, input_index_of_consumer, evidence); - todo!("#1207 PR 4: the readout rule against RequiredAccuracy, else the metric rule") -} - -/// Small relative tolerance for comparing an evaluated bound against a -/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected -/// by floating-point noise. -const SATISFACTION_TOLERANCE: f64 = 1e-9; - -/// The largest evaluated error (or failure probability) that meets a -/// required `limit`: `limit` widened by [`SATISFACTION_TOLERANCE`], so a -/// bound equal to it up to floating-point noise meets it. -pub fn target_limit(limit: f64) -> f64 { - limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON -} - -/// Whether `guarantee`'s evaluated error is within `max_error` and, when -/// `failure_probability` is set, its evaluated failure probability within -/// it ([`target_limit`]); unknown values do not meet. -#[cfg(test)] -pub(crate) fn meets( - guarantee: &AccuracyGuarantee, - max_error: f64, - failure_probability: Option, -) -> bool { - let within = |value: Option, limit: f64| value.is_some_and(|v| v <= target_limit(limit)); - within(guarantee.spec.error.evaluate(), max_error) - && failure_probability - .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A bound equal to the limit up to floating-point noise meets it; a - /// bound one part in a million above it does not. - #[test] - fn target_limit_admits_only_floating_point_noise() { - assert!(0.01 * (1.0 + 1e-12) <= target_limit(0.01)); - assert!(0.01 * (1.0 + 1e-6) > target_limit(0.01)); - assert!(0.0 <= target_limit(0.0)); - } -} diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs deleted file mode 100644 index a79d8b5c2..000000000 --- a/crates/accuracy/src/sizing.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! The statistic a summary is read for. (The predefined parameter lists -//! are checked against the closed-form sizing for an `(ε, δ)` in -//! `guarantee::sizing_oracle`, which is test-only.) - -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::ColumnRef; -use asap_types::summary_type::SketchStatistic; - -/// The statistic a summary is read for to answer `intent`, or `None` when -/// no sketch statistic serves the intent. -pub fn sketch_statistic(intent: &AggIntent) -> Option { - Some(match intent { - AggIntent::Quantile { q, .. } => SketchStatistic::Quantile { q: *q }, - AggIntent::Cardinality { .. } => SketchStatistic::Cardinality, - AggIntent::FrequencyL2 { .. } => SketchStatistic::FrequencyL2, - AggIntent::FrequencyEntropy { .. } => SketchStatistic::FrequencyEntropy, - AggIntent::TopK { k, .. } => SketchStatistic::TopK { k: *k }, - AggIntent::Count => SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }, - _ => return None, - }) -} diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index adc4dc5ab..524548bba 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -778,11 +778,9 @@ fn group_node(table: &mut MemoNodes, group: &MemoGroup) -> usize { fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { OriginOfCandidateForMemoGroup::Pass1 => "pass1", - OriginOfCandidateForMemoGroup::SummaryInput => "summary_input", OriginOfCandidateForMemoGroup::TumblingWindows => "tumbling_windows", OriginOfCandidateForMemoGroup::Rollup => "rollup", OriginOfCandidateForMemoGroup::TopKReuse => "top_k_reuse", - OriginOfCandidateForMemoGroup::WidenedScans => "widened_scans", } .into() } diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index 160d792db..e6faccf28 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -1,7 +1,6 @@ //! The planner stages' output as plain text: `asap_planner --format text`. //! It prints, up to the run's last stage, the frontends' roots and their -//! memo groups (Stage 0), Pass 1's `Memo`, -//! the memo after Pass 2's local window composition and Pass 2's +//! memo groups (Stage 0), Pass 1's `Memo`, Pass 2's //! `LogicalASAPMemoCandidates` (Stage 2's input), Stage 2's //! `PhysicalASAPMemoCandidates` (Stage 3's input), and Stage 3's //! `Selection` with the selected memo choice, each node's cost and each diff --git a/crates/devtools/tests/asap_planner_acceptance.rs b/crates/devtools/tests/asap_planner_acceptance.rs index f9c3068dc..bf8a710a5 100644 --- a/crates/devtools/tests/asap_planner_acceptance.rs +++ b/crates/devtools/tests/asap_planner_acceptance.rs @@ -79,10 +79,7 @@ const SECTIONS: [(&str, &[&str]); 5] = [ ("stage1_pass1", &["Stage 1 Pass 1 output: Memo"]), ( "stage1_pass2", - &[ - "Stage 1 Pass 2, after local window composition", - "Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates", - ], + &["Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates"], ), ( "stage2", diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index 61ee2d8bc..5b5ef3300 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -130,7 +130,7 @@ pub fn validate_ingestion_time_summary( } } -#[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. +#[allow(dead_code)] // Used again once θ comes from the exported node. fn valid_matrix(width: u32, depth: u32) -> bool { // Construction uses the kernel's native row hashing, so no encoded-size // limit applies here. diff --git a/crates/executor/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs index 4b4d58ff1..8863e035f 100644 --- a/crates/executor/src/physical_planner/logical.rs +++ b/crates/executor/src/physical_planner/logical.rs @@ -34,8 +34,7 @@ pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result>), /// A keyed frequency sketch (Count-Min or Count Sketch with a heap) per /// group. - #[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. + #[allow(dead_code)] // Used again once θ comes from the exported node. Keyed { /// An empty state of the family's shape, cloned for each new group /// (boxed: much larger than the other kinds' fields). diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 87d41f5b6..01893a272 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -335,8 +335,11 @@ fn population_fixture_compiles_with_dynamic_identity() { let topk = planner_types::ir::operator::maintained_population::PopulationStatistic::TopK { k: 1 }; let root = Rc::new(with_series_identity(&original).unwrap()); - let selected = planner_types::test_support::population_over_selector(&root, topk, &["job"]); - let logical = planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap(); + let selected = + planner_types::test_support::forwarders::population_over_selector(&root, topk, &["job"]); + let logical = + planner_types::test_support::forwarders::physical_asap_dag_at_query_time(&selected) + .unwrap(); let (source, raw_schema) = logical .nodes .iter() diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index 1d1a2eeab..a55052c4b 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -68,7 +68,6 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: schema(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/deployment_computation.rs b/crates/executor/tests/deployment_computation.rs index 9e552a07e..e36104b0d 100644 --- a/crates/executor/tests/deployment_computation.rs +++ b/crates/executor/tests/deployment_computation.rs @@ -60,9 +60,10 @@ fn population_dag( grouping: &[&str], ) -> PhysicalASAPDAG { let root = promql_rows::with_series_identity(&lower(query)).unwrap(); - let selected = - planner_types::test_support::population_over_selector(&root, statistic, grouping); - planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap() + let selected = planner_types::test_support::forwarders::population_over_selector( + &root, statistic, grouping, + ); + planner_types::test_support::forwarders::physical_asap_dag_at_query_time(&selected).unwrap() } /// The time-series scans: each metric's raw data slot and its schema. diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 0696d2816..dc7726b2b 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -58,7 +58,6 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { lifetime: Lifetime::None, parameters: None, output_schema: output, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index e730108e4..f262878ae 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -160,7 +160,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { }, ])) .clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/node_ids.rs b/crates/executor/tests/node_ids.rs index 961aa096d..34edbb520 100644 --- a/crates/executor/tests/node_ids.rs +++ b/crates/executor/tests/node_ids.rs @@ -19,7 +19,8 @@ fn node_ids_above_u32_are_rejected() { child: values, })) .unwrap(); - let mut dag = planner_types::test_support::physical_asap_dag_at_query_time(&filter).unwrap(); + let mut dag = + planner_types::test_support::forwarders::physical_asap_dag_at_query_time(&filter).unwrap(); let root = dag.roots[0] as u64; assert!(compile(&dag, &[], &[root]).is_ok()); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index b77a7587b..ec9432e08 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -532,7 +532,6 @@ fn bind_physical_asap_dag_before_execution() { lifetime: Default::default(), parameters: None, output_schema: (*schema).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -694,7 +693,6 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -1116,7 +1114,6 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { lifetime: Default::default(), parameters: None, output_schema: (*input).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -1208,7 +1205,6 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Join { kind: JoinKind::Semi, @@ -1316,7 +1312,6 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { output_state: ExecutionDataState::INGESTION_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { operator: BinaryOperator { diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 1413688fb..79be1b280 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -387,7 +387,6 @@ fn global_extrema_bind_with_planner_derived_schema() { lifetime: Default::default(), parameters: None, output_schema: (*output).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -931,7 +930,6 @@ fn planner_node( }) .collect(), ), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -1165,7 +1163,6 @@ fn exact_aggregate_node( lifetime: Default::default(), parameters: None, output_schema: (*schema(&[(name, dtype, false)])).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 64f64962f..a02882402 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -52,7 +52,6 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/planspace_series_identity_heap.rs b/crates/executor/tests/planspace_series_identity_heap.rs index 71bc6302a..d4f9ca217 100644 --- a/crates/executor/tests/planspace_series_identity_heap.rs +++ b/crates/executor/tests/planspace_series_identity_heap.rs @@ -44,7 +44,7 @@ fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { } fn carries_identity(root: &Rc) -> bool { - planner_types::test_support::physical_asap_dag_at_query_time(root) + planner_types::test_support::forwarders::physical_asap_dag_at_query_time(root) .unwrap() .nodes .iter() diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 938cf0b5a..39486bb2d 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -49,7 +49,6 @@ fn view_over_samples(family: FieldDataType, scan: usize) -> [PhysicalASAPDAGNode lifetime: Default::default(), parameters: None, output_schema, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -138,7 +137,6 @@ fn try_state_dag( ), parameters: None, output_schema: logical_schema(family.clone()), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -154,7 +152,6 @@ fn try_state_dag( lifetime: Default::default(), parameters: None, output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -175,7 +172,6 @@ fn try_state_dag( ), parameters: None, output_schema: logical_schema(target), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index d2c03d033..fb36382d1 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -82,7 +82,6 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { lifetime: Default::default(), parameters: None, output_schema: (*schema).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -422,7 +421,6 @@ fn stored_series_evaluations_support_filters_and_sets() { 5 | 6 => sample_schema(), _ => value_schema.clone(), }, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 10bc11ffe..25d3fe675 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -87,7 +87,6 @@ fn plan( lifetime: Default::default(), parameters: None, output_schema: (**schema).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -510,7 +509,6 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -845,7 +843,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: output_schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 4c385fca6..9570e9fcb 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -318,7 +318,6 @@ fn merge_over_a_stored_tumbling_window( output_state, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index c624a6d9e..48b458694 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -55,7 +55,6 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -126,7 +125,6 @@ fn summary_projection_survives_recovery() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -156,7 +154,6 @@ fn summary_projection_survives_recovery() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -265,7 +262,6 @@ fn summary_merge_without_inputs_is_rejected() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index a05a19649..ef24d850c 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -83,7 +83,7 @@ fn window_over_rows( child, })) .unwrap(); - planner_types::test_support::physical_asap_dag_at_query_time(&range).unwrap() + planner_types::test_support::forwarders::physical_asap_dag_at_query_time(&range).unwrap() } /// `(instance, ts, value)` samples of `latency_ms` as a store's raw data. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 49241d6a5..fcc1ba2fd 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -181,7 +181,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() lifetime: Default::default(), parameters: None, output_schema: (*state).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index c933d4857..4deb6e593 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -103,7 +103,7 @@ pub mod physical_export { /// Export the physical DAG of `root`. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - asap_types::test_support::physical_asap_dag_at_query_time(root) + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(root) .expect("physical ASAP DAG export failed") } } diff --git a/crates/integration-tests/tests/crate_graph.rs b/crates/integration-tests/tests/crate_graph.rs index 1dcb0ec12..c948add4b 100644 --- a/crates/integration-tests/tests/crate_graph.rs +++ b/crates/integration-tests/tests/crate_graph.rs @@ -1,4 +1,4 @@ -//! The workspace crate graph follows #1054's target graph (its rule 4). +//! The workspace crate graph follows the target crate graph (its rule 4). //! //! The test reads `cargo metadata --format-version 1 --no-deps` and checks every //! `[dependencies]` edge between two workspace crates, and every @@ -16,7 +16,7 @@ //! - the **facade** (asap-planner) may depend on every stage crate and every //! shared crate; //! - the **executor** is a runtime, not a stage; it depends only on asap-types -//! (#1016 decides where it belongs); +//! (where it belongs is not decided); //! - **devtools** and **integration-tests** may depend on anything, and no //! other crate may depend on them. //! @@ -100,8 +100,8 @@ type Exception = (&'static str, &'static str); /// The forbidden `[dependencies]` edges the workspace keeps today. const EXCEPTIONS: &[Exception] = &[ // `plan_installable` offers only the summaries the Planner's own executor - // implements. #1016 decides where that kernel list and the executor - // belong; this exception goes away with it. + // implements. Where that kernel list and the executor belong is + // not decided; this exception goes away with it. ("asap-planner", "asap-executor"), ]; @@ -159,9 +159,7 @@ fn crate_graph_violations(metadata: &Value) -> Vec { continue; }; if !exceptions.contains(&(from, to)) { - violations.push(format!( - "forbidden {section} edge {from} -> {to}: {rule} (#1054)" - )); + violations.push(format!("forbidden {section} edge {from} -> {to}: {rule}")); } } } @@ -222,7 +220,7 @@ fn checker_rejects_an_edge_between_stage_crates() { crate_graph_violations(&metadata), vec![ "forbidden [dependencies] edge asap-stage3-plan-selection -> \ - asap-stage2-physical-optimization: stage crates never depend on each other (#1054)" + asap-stage2-physical-optimization: stage crates never depend on each other" .to_string() ] ); @@ -255,7 +253,7 @@ fn checker_rejects_a_dev_dependency_between_stage_crates() { vec![ "forbidden [dev-dependencies] edge asap-stage1-pass1-per-computation-candidates -> \ asap-stage0-memo: a stage crate's tests never call another stage crate, \ - directly or through the facade (#1054)" + directly or through the facade" .to_string() ] ); @@ -274,7 +272,7 @@ fn checker_rejects_a_frontend_dev_dependency_on_the_facade() { vec![ "forbidden [dev-dependencies] edge asap-frontend-promql -> asap-planner: \ a stage crate's tests never call another stage crate, directly or through \ - the facade (#1054)" + the facade" .to_string() ] ); @@ -292,7 +290,7 @@ fn checker_rejects_an_unclassified_crate_and_shared_model_edges() { crate_graph_violations(&metadata), vec![ "forbidden [dependencies] edge asap-accuracy -> asap-stage1-pass1-per-computation-candidates: \ - asap-accuracy and asap-cost depend only on asap-types (#1054)" + asap-accuracy and asap-cost depend only on asap-types" .to_string(), "workspace crate `asap-new-crate` has no role: add it to `role_of` in \ crates/integration-tests/tests/crate_graph.rs" diff --git a/crates/integration-tests/tests/filtered_aggregates.rs b/crates/integration-tests/tests/filtered_aggregates.rs index c95b9d7f8..970abf26a 100644 --- a/crates/integration-tests/tests/filtered_aggregates.rs +++ b/crates/integration-tests/tests/filtered_aggregates.rs @@ -57,7 +57,7 @@ fn rows() -> Vec> { /// Run the logical DAG `root` over [`rows`]; rows come back shown and sorted. fn execute(root: &Rc) -> Vec { - let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); + let dag = asap_types::test_support::forwarders::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() diff --git a/crates/integration-tests/tests/finer_aggregate_reuse.rs b/crates/integration-tests/tests/finer_aggregate_reuse.rs index 9ce0534a7..9f893a59c 100644 --- a/crates/integration-tests/tests/finer_aggregate_reuse.rs +++ b/crates/integration-tests/tests/finer_aggregate_reuse.rs @@ -1,64 +1,22 @@ //! A coarse aggregate read from a finer one through a compensating filter and, -//! when its groups are coarser, a rollup of values or states (#950). -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use asap_devtools::planning_stages::run_search_over; - -use serde_json::json; -use std::path::Path; +//! when its groups are coarser, a rollup of values or states. +use crate::planning_common::{self, fuzz_workload, with_queries}; +use asap_devtools::planner_fuzz::FuzzWorkload; const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; const WINDOW: &str = "ts > now() - INTERVAL '5' MINUTE"; -/// The #902 workload (5-minute window, every 10 s, ε = 0.01) with `queries`. +/// The percentile workload (5-minute window, every 10 s, ε = 0.01) with `queries`. fn workload(queries: &[String]) -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/sharing_issue_fixtures/902.json"), + with_queries( + fuzz_workload("percentile_filtered_and_by_city", true), + queries, ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":true}, - {"name":"host","type":"utf8","nullable":true}, - {"name":"value","type":"float64","nullable":true} - ]})]; - let template = w.queries[0].clone(); - w.queries = queries - .iter() - .map(|query| { - let mut entry = template.clone(); - entry.entry.query.0 = query.clone(); - entry - }) - .collect(); - w } /// The selected plan's cost per second for `queries`. fn cost(queries: &[String]) -> f64 { - use asap_cost::CostInputs; - use asap_planner::{PlanSearchSettings, PlanningModels}; - let w = workload(queries); - let roots = Planner::new(FuzzSettings::default()).lower(&w).unwrap(); - let planning = w.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - let settings = CostInputs::default(); - let deployment = w.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment); - let run = run_search_over( - roots, - &entries, - &w.data, - models, - PlanSearchSettings::default(), - ) - .unwrap(); - let (plan, _) = run.run.selection.as_ref().unwrap(); - let value = - asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); - println!("cost={value}"); - value + planning_common::cost(&workload(queries)) } /// Example B end to end: adding the filtered sum to the finer sums costs diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 3fe8f7d61..166a87fa4 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -1,4 +1,4 @@ -//! Cross-frontend evaluation-time semantics (issues #46 and #184). +//! Cross-frontend evaluation-time semantics. use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::fixtures::lower_promql_root; diff --git a/crates/integration-tests/tests/identical_exact_queries.rs b/crates/integration-tests/tests/identical_exact_queries.rs deleted file mode 100644 index 95b550dc3..000000000 --- a/crates/integration-tests/tests/identical_exact_queries.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! Original issue workloads against the production candidate-connection planner. -use asap_cost::CostInputs; -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use asap_devtools::planning_stages::run_search_over; -use asap_planner::{PlanSearchSettings, PlanningModels}; -use serde_json::json; -use std::path::Path; - -fn workload() -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/sharing_issue_fixtures/922.json"), - ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":false}, - {"name":"host","type":"utf8","nullable":false}, - {"name":"value","type":"float64","nullable":false} - ]})]; - w -} - -fn cost(w: &FuzzWorkload) -> f64 { - let roots = Planner::new(FuzzSettings::default()).lower(w).unwrap(); - let planning = w.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - let settings = CostInputs::default(); - let deployment = w.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment); - let run = run_search_over( - roots, - &entries, - &w.data, - models, - PlanSearchSettings::default(), - ) - .unwrap(); - let (plan, _) = run.run.selection.as_ref().unwrap(); - let value = - asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); - println!("cost={value}"); - value -} - -/// #922: two identical exact percentile queries pay for their result once. -#[test] -fn identical_exact_percentiles_share_work() { - let related = workload(); - let mut base = related.clone(); - base.queries.truncate(1); - let base_cost = cost(&base); - let related_cost = cost(&related); - assert!( - related_cost <= base_cost * 1.001, - "{related_cost} vs {base_cost}" - ); -} - -/// Different ranks share the source window but still compute their own values. -#[test] -fn different_exact_percentiles_share_the_input_window() { - let mut related = workload(); - related.queries[1].entry.query.0 = related.queries[1].entry.query.0.replace("0.99", "0.5"); - let related_cost = cost(&related); - assert!(related_cost <= 0.0883, "{related_cost}"); -} diff --git a/crates/integration-tests/tests/independent_plan_seed.rs b/crates/integration-tests/tests/independent_plan_seed.rs index 4faffc3a5..5b7371431 100644 --- a/crates/integration-tests/tests/independent_plan_seed.rs +++ b/crates/integration-tests/tests/independent_plan_seed.rs @@ -1,60 +1,22 @@ //! The joint search never returns a plan costlier than planning each query //! on its own: each query's own plan, combined, is evaluated first. -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use serde_json::json; -use std::path::Path; +use crate::planning_common::{self, fuzz_workload, with_queries}; +use asap_devtools::planner_fuzz::FuzzWorkload; const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; const W5: &str = "ts > now() - INTERVAL '5' MINUTE"; -/// `queries` in the #902 SQL workload (`metrics`, every 10 s). +/// `queries` in the percentile workload over `metrics`, every 10 s. fn sql(queries: &[String]) -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/sharing_issue_fixtures/902.json"), + with_queries( + fuzz_workload("percentile_filtered_and_by_city", true), + queries, ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":true}, - {"name":"host","type":"utf8","nullable":true}, - {"name":"value","type":"float64","nullable":true} - ]})]; - let template = w.queries[0].clone(); - w.queries = queries - .iter() - .map(|query| { - let mut entry = template.clone(); - entry.entry.query.0 = query.clone(); - entry - }) - .collect(); - w } -/// The selected plan's cost per second, with the default 30 s budget. +/// The selected plan's cost per second for `queries`. fn cost(queries: &[String]) -> f64 { - use asap_cost::CostInputs; - use asap_devtools::planning_stages::run_search_over; - use asap_planner::{PlanSearchSettings, PlanningModels}; - let w = sql(queries); - let roots = Planner::new(FuzzSettings::default()).lower(&w).unwrap(); - let planning = w.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - let settings = CostInputs::default(); - let deployment = w.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment); - let run = run_search_over( - roots, - &entries, - &w.data, - models, - PlanSearchSettings::default(), - ) - .unwrap(); - let (plan, _) = run.run.selection.as_ref().unwrap(); - asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings) + planning_common::cost(&sql(queries)) } /// Asserts the joint plan of `queries` costs no more than their separate diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index 48e72c8c1..c49abdef5 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -107,7 +107,8 @@ fn fixture() -> Fixture { QueryRoot::Operator(estimate(0.99)), ]; let (dag, operator_node_of_id) = - asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots).unwrap(); + asap_types::test_support::forwarders::physical_asap_dag_at_query_time_with_node_ids(&roots) + .unwrap(); let mut tumbling_window_ids = vec![0; 5]; let (mut merge, mut p50, mut p99) = (0, 0, 0); for node in &dag.nodes { diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index bc74bcd3d..2f36e7401 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -4,6 +4,7 @@ mod physical_common; mod planner_layering_common; +mod planning_common; mod aggregate; mod binary_op; @@ -11,7 +12,6 @@ mod crate_graph; mod filtered_aggregates; mod finer_aggregate_reuse; mod frontend_timestamps; -mod identical_exact_queries; mod independent_plan_seed; mod joint_search_equivalence; mod kll_tumbling_window_execution; @@ -20,14 +20,15 @@ mod nested; mod operator_design_examples; mod operator_sharing; mod pass1_candidates; -mod pass1_candidates_sql; +mod pass2_promql; +mod pass2_sql; mod per_series_exact_execution; +mod plan_sharing; mod production_rollup_workloads; mod promql_to_post_asap; mod rollup_controlled; mod scan; mod schema; -mod sharing_issue_regressions; mod sql_cardinality; mod sql_frequency; mod sql_to_post_asap; diff --git a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json index 53ef51dfd..5c762eb6c 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json @@ -1 +1 @@ -{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "filter": null, "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "filter": null, "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} +{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "filter": null, "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "filter": null, "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_before_view_coverage.json similarity index 64% rename from crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json rename to crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_before_view_coverage.json index f822d6b70..f0abf8bbe 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_before_view_coverage.json @@ -1,10 +1,10 @@ [ -{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, -{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, -{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, -{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, -{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, -{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} +{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, +{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, +{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, +{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, +{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, +{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} ] diff --git a/crates/integration-tests/tests/materialized_views.rs b/crates/integration-tests/tests/materialized_views.rs index 1633e5691..17eecc9a7 100644 --- a/crates/integration-tests/tests/materialized_views.rs +++ b/crates/integration-tests/tests/materialized_views.rs @@ -1,5 +1,5 @@ //! What a runtime derives from an installable plan's DAG alone: its -//! materialized views and its raw-data retention (issue #970). +//! materialized views and its raw-data retention. use std::collections::HashSet; @@ -14,7 +14,7 @@ use asap_types::ir::{NonASAPOp, Operator}; use serde_json::Value; /// A plan file of wire version 10 (assembled from the parts of a plan -/// written before #970: the versioned DAG envelope, `queries`, +/// written before views were derived from the DAG: the versioned DAG envelope, `queries`, /// `stored_outputs`, `query_time_raw_windows`, `selection`; its DAG nodes /// carry today's node fields, e.g. `lifetime`) is refused for its version, /// not for a missing field; its DAG with today's version reads back. @@ -45,7 +45,7 @@ fn a_plan_file_of_an_older_wire_version_is_refused_by_version() { assert_eq!(read, current); } -/// For installable plans captured from `main` before #970 (each the plan +/// For installable plans captured before views were derived from the DAG (each the plan /// file's DAG and its `stored_outputs` field), `materialized_views()` of /// that DAG, with each view's `retention_ms` on its node, equals /// `stored_outputs` field for field after the renames: `built_at` → @@ -57,7 +57,7 @@ fn a_plan_file_of_an_older_wire_version_is_refused_by_version() { #[test] fn materialized_views_equal_the_stored_outputs_main_listed() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/materialized_view_fixtures/stored_outputs_on_main.json"); + .join("tests/materialized_view_fixtures/stored_outputs_before_view_coverage.json"); let cases: Vec = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); assert!(!cases.is_empty()); let mut refreshes = HashSet::new(); diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 80ceb6abf..c980d7f92 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -1,4 +1,4 @@ -//! #511 examples: source text → unified dag → summary rewrite → flat export. +//! Operator IR examples: source text → unified dag → summary rewrite → flat export. use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; @@ -41,7 +41,8 @@ async fn sql_filter_projection_example() { assert!( matches!(root.expect_non_asap(),NonASAPOp::Project { cols,.. } if matches!(cols[0].expr,ScalarExpr::Arithmetic { .. })) ); - let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); + let wire = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&root).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), OperatorNode::reachable(&root).len()); } @@ -83,15 +84,10 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { })) .unwrap(), ); - let finalize = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - }), - node.schema.clone(), - ) - .with_guarantee(None), - ); + let finalize = std::rc::Rc::new(OperatorNode::with_schema( + asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + node.schema.clone(), + )); return finalize; } Rc::new(node.with_new_children(rewrite).unwrap()) @@ -106,7 +102,8 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { assert!(dag .iter() .any(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))); - let wire = asap_types::test_support::physical_asap_dag_at_query_time(&rewritten).unwrap(); + let wire = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&rewritten).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), dag.len()); } @@ -121,7 +118,8 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { let root = lower_sql(query, &catalog()).await.unwrap(); root.validate_structure().unwrap(); assert!(root.children().len() > 1); - let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); + let wire = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&root).unwrap(); assert!(wire .edges .iter() @@ -153,7 +151,8 @@ async fn sql_sum_example_executes_with_sql_null_semantics() { let NonASAPOp::Scan { source, .. } = logical_scan.expect_non_asap() else { panic!() }; - let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); + let wire = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&root).unwrap(); let scan = wire .nodes .iter() diff --git a/crates/integration-tests/tests/operator_sharing.rs b/crates/integration-tests/tests/operator_sharing.rs index 22dec7ea8..fbb00acb0 100644 --- a/crates/integration-tests/tests/operator_sharing.rs +++ b/crates/integration-tests/tests/operator_sharing.rs @@ -1,4 +1,4 @@ -//! Acceptance tests for operator sharing (issue #468): one operator IR +//! Acceptance tests for operator sharing: one operator IR //! before and after ASAP optimization, so non-ASAP operators sit both above //! and below summary operators, can share inputs with them, and can carry //! summaries below set operators. @@ -68,7 +68,7 @@ fn is_scan(node: &OperatorNode) -> bool { matches!(node.non_asap(), Some(NonASAPOp::Scan { .. })) } -// #468 problem 2: the exact aggregate and the sketch read one shared Scan +// Problem 2: the exact aggregate and the sketch read one shared Scan // (`Rc::ptr_eq`), not two copies. #[ignore = "waits for the binding rule splitting multi-measure aggregates"] #[tokio::test] @@ -101,7 +101,7 @@ async fn exact_aggregate_and_sketch_share_one_scan() { ); } -// #468 problem 3: a summary can sit below a set operator — each side of the +// Problem 3: a summary can sit below a set operator — each side of the // UNION ALL holds its own SummaryEstimate. #[tokio::test] #[ignore = "plan selection picks the exact plan: a query-time summary on each side never costs less than the exact aggregate until Pass 2 shares it (#580)"] diff --git a/crates/integration-tests/tests/pass1_candidates.rs b/crates/integration-tests/tests/pass1_candidates.rs index 9e4c9930d..8457b6506 100644 --- a/crates/integration-tests/tests/pass1_candidates.rs +++ b/crates/integration-tests/tests/pass1_candidates.rs @@ -1,6 +1,7 @@ //! Frontends → Pass 1's memo: every candidate Pass 1 adds outputs its //! group's schema (`Memo::add_candidate` checks it), so every way to compute -//! a query keeps its output columns (names, types and nullability). +//! a query keeps its output columns (names, types and nullability); where +//! each SQL `HAVING` is applied and what a Top-K heap sketch ranks. use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_stage1_pass1_per_computation_candidates::pass1_memo; @@ -9,7 +10,7 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::FieldDataType; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, ScalarExpr}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, Predicate, QueryRoot, ScalarExpr}; use asap_types::stages::{Memo, MemoGroupId}; use asap_types::summary_type::{SketchAlgorithm, SummaryInputExpr, SummaryUpdate}; use asap_types::workload::{ @@ -521,3 +522,196 @@ async fn sql_count_of_a_nullable_column_counts_its_values() { assert!(reads_only(update, "v"), "{updates:?}"); } } + +/// Pass 1's memo of `sql` over `flows(src, bytes)`. +async fn pass1_flows(sql: &str) -> Memo { + pass1_flows_over( + vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ], + sql, + ) + .await +} + +/// Pass 1's memo of `sql` over `flows(fields)`. +async fn pass1_flows_over(fields: Vec, sql: &str) -> Memo { + pass1_sql( + &SqlCatalog::new().with_table("flows", Schema::new(fields)), + sql, + ) + .await +} + +/// Whether some candidate applies a `HAVING` below the aggregate: a +/// `Filter` group over the aggregate's input, which a state build reads. +fn having_pushed_below(memo: &Memo) -> bool { + let groups = memo.groups(); + groups.iter().any(|group| { + let Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) = &group.expression else { + return false; + }; + groups.iter().any(|filter| { + matches!(&filter.expression, Operator::NonASAP(NonASAPOp::Filter { child: input, .. }) if input == child) + }) + }) +} + +/// A key-only `HAVING` is pushed below the aggregate. +#[tokio::test] +async fn key_only_having_is_pushed_below() { + let memo = pass1_flows("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'").await; + assert!(having_pushed_below(&memo)); +} + +/// A `HAVING` over an aggregate value stays above the output. +#[tokio::test] +async fn value_having_stays_above() { + let memo = + pass1_flows("SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10").await; + assert!(!having_pushed_below(&memo)); +} + +/// A global aggregate's `HAVING`, even a constant one, stays above the output. +#[tokio::test] +async fn global_constant_having_stays_above() { + let memo = pass1_flows("SELECT count(*) FROM flows HAVING false").await; + assert!(!having_pushed_below(&memo)); +} + +/// A `HAVING` that reads a key and a value pushes its key conjunct below +/// the aggregate and keeps its value conjunct above: some candidate of the +/// `HAVING`'s `Filter` group filters by `count(*) > 10` alone. +#[tokio::test] +async fn mixed_having_pushes_its_key_conjunct_below() { + let memo = pass1_flows( + "SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a' AND count(*) > 10", + ) + .await; + assert!(having_pushed_below(&memo)); + let (filter, value_conjunct) = memo + .groups() + .iter() + .find_map(|group| match &group.expression { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::BoolAnd(conjuncts)), + .. + }) => Some((group, conjuncts[1].clone())), + _ => None, + }) + .expect("the HAVING's Filter group"); + assert!(filter.candidates.iter().any(|candidate| matches!( + &candidate.expression.operator, + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) if pred.0 == value_conjunct + ))); +} + +/// The item and weight columns of every heap sketch state group in Pass 1's +/// memo of `sql` over `flows(key: key_type, bytes)`. +async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, String)> { + heap_updates_over( + vec![ + Field::plain(key, key_type, false), + Field::plain("bytes", DataType::Float64, false), + ], + sql, + ) + .await +} + +/// [`heap_updates`] over `flows(fields)`. +async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String)> { + let memo = pass1_flows_over(fields, sql).await; + let name = |expr: &SummaryInputExpr| match expr { + SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), + // The ranked value when it is the child's one sample value. + SummaryInputExpr::Column(ColumnRef::SampleValue) => "".into(), + SummaryInputExpr::Tuple(items) => match items.as_slice() { + [SummaryInputExpr::Column(ColumnRef::Named(name))] => name.clone(), + other => panic!("one item column: {other:?}"), + }, + SummaryInputExpr::Constant(value) => value.to_string(), + other => panic!("a named column or constant: {other:?}"), + }; + memo.groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: + FieldDataType::Sketch( + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, + ), + input, + .. + }) => Some((name(input.item.as_ref().unwrap()), name(&input.weight))), + _ => None, + }) + .collect() +} + +const COUNT_RANKED_TOPK: &str = + "SELECT KEY, count(*) FROM flows GROUP BY KEY ORDER BY count(*) DESC LIMIT 10"; + +/// An Int64 key is not mistaken for the ranked value: both heap sketches +/// are built over raw rows, keyed by the key with unit weight. +#[tokio::test] +async fn topk_heap_over_an_int64_key_counts_raw_rows() { + let updates = heap_updates( + "port", + DataType::Int64, + &COUNT_RANKED_TOPK.replace("KEY", "port"), + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "port" && weight == "1"), + "{updates:?}" + ); +} + +/// A key named `value` is not the ranked value: the heap sketches are +/// fed raw rows with unit weight. +#[tokio::test] +async fn topk_heap_over_a_key_named_value_counts_raw_rows() { + let updates = heap_updates( + "value", + DataType::Utf8, + &COUNT_RANKED_TOPK.replace("KEY", "value"), + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "value" && weight == "1"), + "{updates:?}" + ); +} + +/// SQL's per-group top-k idiom (`ROW_NUMBER() OVER (PARTITION BY city +/// ORDER BY COUNT(*) DESC) … WHERE rn <= 5`) is a heavy-hitter top-k, as +/// PromQL `topk by (city)` is: both heap sketches are offered, keyed by the +/// host and fed raw rows with unit weight. +#[tokio::test] +async fn rownumber_per_group_topk_offers_heap_sketches() { + let updates = heap_updates_over( + vec![ + Field::plain("city", DataType::Utf8, false), + Field::plain("host", DataType::Utf8, false), + ], + "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, ROW_NUMBER() OVER \ + (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM flows GROUP BY city, host) \ + WHERE rn <= 5", + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "host" && weight == "1"), + "{updates:?}" + ); +} diff --git a/crates/integration-tests/tests/pass1_candidates_sql.rs b/crates/integration-tests/tests/pass1_candidates_sql.rs deleted file mode 100644 index 17db96566..000000000 --- a/crates/integration-tests/tests/pass1_candidates_sql.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! SQL → Pass 1 memo candidates: where each `HAVING` is applied, the -//! guarantees composed over estimates, and what a Top-K heap sketch ranks. - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_stage1_pass1_per_computation_candidates::pass1_memo; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, Predicate, QueryRoot, ScalarExpr}; -use asap_types::stages::Memo; -use asap_types::summary_type::{SketchAlgorithm, SummaryInputExpr}; - -/// Pass 1's memo of `sql` over `flows(src, bytes)`. -async fn pass1(sql: &str) -> Memo { - pass1_over( - vec![ - Field::plain("src", DataType::Utf8, false), - Field::plain("bytes", DataType::Float64, false), - ], - sql, - ) - .await -} - -/// Pass 1's memo of `sql` over `flows(fields)`. -async fn pass1_over(fields: Vec, sql: &str) -> Memo { - let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); - let root = lower_sql(sql, &catalog) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - let memo = asap_stage0_memo::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(); - pass1_memo(memo, None).unwrap_or_else(|e| panic!("Pass 1 failed for {sql:?}: {e}")) -} - -/// Whether some candidate applies a `HAVING` below the aggregate: a -/// `Filter` group over the aggregate's input, which a state build reads. -fn having_pushed_below(memo: &Memo) -> bool { - let groups = memo.groups(); - groups.iter().any(|group| { - let Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) = &group.expression else { - return false; - }; - groups.iter().any(|filter| { - matches!(&filter.expression, Operator::NonASAP(NonASAPOp::Filter { child: input, .. }) if input == child) - }) - }) -} - -/// A key-only `HAVING` is pushed below the aggregate. -#[tokio::test] -async fn key_only_having_is_pushed_below() { - let memo = pass1("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'").await; - assert!(having_pushed_below(&memo)); -} - -/// A `HAVING` over an aggregate value stays above the output. -#[tokio::test] -async fn value_having_stays_above() { - let memo = pass1("SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10").await; - assert!(!having_pushed_below(&memo)); -} - -/// A global aggregate's `HAVING`, even a constant one, stays above the output. -#[tokio::test] -async fn global_constant_having_stays_above() { - let memo = pass1("SELECT count(*) FROM flows HAVING false").await; - assert!(!having_pushed_below(&memo)); -} - -/// A `HAVING` that reads a key and a value pushes its key conjunct below -/// the aggregate and keeps its value conjunct above: some candidate of the -/// `HAVING`'s `Filter` group filters by `count(*) > 10` alone. -#[tokio::test] -async fn mixed_having_pushes_its_key_conjunct_below() { - let memo = - pass1("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a' AND count(*) > 10") - .await; - assert!(having_pushed_below(&memo)); - let (filter, value_conjunct) = memo - .groups() - .iter() - .find_map(|group| match &group.expression { - Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::BoolAnd(conjuncts)), - .. - }) => Some((group, conjuncts[1].clone())), - _ => None, - }) - .expect("the HAVING's Filter group"); - assert!(filter.candidates.iter().any(|candidate| matches!( - &candidate.expression.operator, - Operator::NonASAP(NonASAPOp::Filter { pred, .. }) if pred.0 == value_conjunct - ))); -} - -/// The item and weight columns of every heap sketch state group in Pass 1's -/// memo of `sql` over `flows(key: key_type, bytes)`. -async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, String)> { - heap_updates_over( - vec![ - Field::plain(key, key_type, false), - Field::plain("bytes", DataType::Float64, false), - ], - sql, - ) - .await -} - -/// [`heap_updates`] over `flows(fields)`. -async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String)> { - let memo = pass1_over(fields, sql).await; - let name = |expr: &SummaryInputExpr| match expr { - SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), - // The ranked value when it is the child's one sample value. - SummaryInputExpr::Column(ColumnRef::SampleValue) => "".into(), - SummaryInputExpr::Tuple(items) => match items.as_slice() { - [SummaryInputExpr::Column(ColumnRef::Named(name))] => name.clone(), - other => panic!("one item column: {other:?}"), - }, - SummaryInputExpr::Constant(value) => value.to_string(), - other => panic!("a named column or constant: {other:?}"), - }; - memo.groups() - .iter() - .filter_map(|group| match &group.expression { - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: - FieldDataType::Sketch( - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, - ), - input, - .. - }) => Some((name(input.item.as_ref().unwrap()), name(&input.weight))), - _ => None, - }) - .collect() -} - -const COUNT_RANKED_TOPK: &str = - "SELECT KEY, count(*) FROM flows GROUP BY KEY ORDER BY count(*) DESC LIMIT 10"; - -/// An Int64 key is not mistaken for the ranked value: both heap sketches -/// are built over raw rows, keyed by the key with unit weight. -#[tokio::test] -async fn topk_heap_over_an_int64_key_counts_raw_rows() { - let updates = heap_updates( - "port", - DataType::Int64, - &COUNT_RANKED_TOPK.replace("KEY", "port"), - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "port" && weight == "1"), - "{updates:?}" - ); -} - -/// A key named `value` is not the ranked value: the heap sketches are -/// fed raw rows with unit weight. -#[tokio::test] -async fn topk_heap_over_a_key_named_value_counts_raw_rows() { - let updates = heap_updates( - "value", - DataType::Utf8, - &COUNT_RANKED_TOPK.replace("KEY", "value"), - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "value" && weight == "1"), - "{updates:?}" - ); -} - -/// SQL's per-group top-k idiom (`ROW_NUMBER() OVER (PARTITION BY city -/// ORDER BY COUNT(*) DESC) … WHERE rn <= 5`) is a heavy-hitter top-k, as -/// PromQL `topk by (city)` is: both heap sketches are offered, keyed by the -/// host and fed raw rows with unit weight. -#[tokio::test] -async fn rownumber_per_group_topk_offers_heap_sketches() { - let updates = heap_updates_over( - vec![ - Field::plain("city", DataType::Utf8, false), - Field::plain("host", DataType::Utf8, false), - ], - "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, ROW_NUMBER() OVER \ - (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM flows GROUP BY city, host) \ - WHERE rn <= 5", - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "host" && weight == "1"), - "{updates:?}" - ); -} diff --git a/crates/integration-tests/tests/pass2_promql.rs b/crates/integration-tests/tests/pass2_promql.rs new file mode 100644 index 000000000..f0c41bfda --- /dev/null +++ b/crates/integration-tests/tests/pass2_promql.rs @@ -0,0 +1,4 @@ +//! PromQL → Pass 2, through production lowering. +mod accuracy_and_grids; +mod exact_only_groups; +mod rollup; diff --git a/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs b/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs new file mode 100644 index 000000000..77f0afff8 --- /dev/null +++ b/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs @@ -0,0 +1,114 @@ +//! PromQL → Pass 2's error contracts and evaluation grids, through +//! production lowering: the frontend's `Project`s of a function of the +//! value keep a readout's error bounded, and a subquery's resolution is the +//! evaluation grid of the groups below it. +use crate::planning_common::promql_pass2; +use asap_types::ir::operator::asap::summary_type_of_operator; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, MemoGroupId}; +use serde_json::{json, Value}; + +/// Pass 2's output for `query` at `accuracy`. +fn pass2(query: &str, accuracy: Value) -> LogicalASAPMemoCandidates { + promql_pass2(&[(query, accuracy)]) +} + +/// The sketch readouts (`SummaryEstimate` over a sketch state) that +/// survive Pass 2. +fn live_sketch_readouts(logical: &LogicalASAPMemoCandidates) -> usize { + let memo = &logical.memo; + let mut live = 0; + for (group, memo_group) in memo.groups().iter().enumerate() { + for (index_in_group, candidate) in memo_group.candidates.iter().enumerate() { + let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = + &candidate.expression.operator + else { + continue; + }; + let sketch = matches!( + summary_type_of_operator(&memo.groups()[summary_input.0 as usize].expression), + Some(FieldDataType::Sketch(_)) + ); + let id = CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }; + if sketch && logical.pruned.iter().all(|pruned| pruned.candidate != id) { + live += 1; + } + } + } + live +} + +/// The frontend lowers `abs(v)`, `-v` and `v / 2` to a `Project` that +/// replaces the value and passes the series identity through +/// `promql_drop_metric_name`. Its contract judges only the value: `abs` +/// and the negation keep an absolute error, scaling by a constant a +/// relative one, so a sketch readout below each still serves the query, as +/// it does for the bare readout. +#[test] +fn a_function_of_the_value_keeps_the_readouts_below_it() { + let absolute = json!({"max_error": 0.05, "accuracy_metric": "absolute"}); + let relative = json!({"max_error": 0.05, "accuracy_metric": "relative"}); + for (query, accuracy) in [ + ("count_over_time(m[1m])", &absolute), + ("abs(count_over_time(m[1m]))", &absolute), + ("-count_over_time(m[1m])", &absolute), + ("quantile_over_time(0.9, m[5m]) / 2", &relative), + ("quantile_over_time(0.9, m[5m]) * 2", &relative), + ] { + let logical = pass2(query, accuracy.clone()); + assert!(live_sketch_readouts(&logical) > 0, "{query}"); + } +} + +/// The groups whose expression is a tumbling-window chain, with its width. +fn chain_widths(logical: &LogicalASAPMemoCandidates) -> Vec { + logical + .memo + .groups() + .iter() + .filter_map(|group| match group.expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(tumbling_window_width_ms), + _ => None, + }) + .collect() +} + +/// `quantile_over_time(0.9, rate(c[2m])[10m:30s])` read every 60 s +/// evaluates the inner `rate` every 30 s: its groups are on the 30 s grid +/// and read only chains whose width divides 30 s. Without a resolution the +/// inner step is the server's default, unknown: no grid, no chain. +#[test] +fn a_subquery_s_resolution_is_the_grid_of_the_groups_below_it() { + let rank = json!({"max_error": 0.01, "accuracy_metric": "rank"}); + let logical = pass2( + "quantile_over_time(0.9, rate(c[2m])[10m:30s])", + rank.clone(), + ); + let subquery = logical + .memo + .groups() + .iter() + .position(|group| { + matches!( + group.expression, + Operator::NonASAP(NonASAPOp::PromqlSubquery { .. }) + ) + }) + .expect("a subquery group"); + let child = logical.memo.groups()[subquery].expression.children()[0].0 as usize; + let period = |group: usize| logical.evaluation_grid_of_group[group].map(|grid| grid.period_ms); + assert_eq!(period(subquery), Some(60_000)); + assert_eq!(period(child), Some(30_000)); + let widths = chain_widths(&logical); + assert!(!widths.is_empty()); + assert!(widths.iter().all(|width| 30_000 % width == 0), "{widths:?}"); + let without_resolution = pass2("max_over_time(rate(c[2m])[10m:])", rank); + assert_eq!(chain_widths(&without_resolution), Vec::::new()); +} diff --git a/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs b/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs new file mode 100644 index 000000000..aa6461292 --- /dev/null +++ b/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs @@ -0,0 +1,119 @@ +//! PromQL → the groups each query may read only exactly, through +//! production lowering: a query never reads a sketch readout whose error +//! metric its target does not allow, and a dead candidate makes no group +//! exact-only. +use crate::planning_common::promql_pass2; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, MemoGroupId}; +use asap_types::summary_type::SketchAlgorithm; +use serde_json::json; + +/// The readout groups (`SummaryEstimate`) over a state of `algorithm`. +fn readout_groups( + logical: &LogicalASAPMemoCandidates, + algorithm: SketchAlgorithm, +) -> Vec { + let memo = &logical.memo; + (0..memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .filter(|group| match &memo.groups()[group.0 as usize].expression { + Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => memo.groups() + [summary_input.0 as usize] + .group_properties + .schema + .fields + .iter() + .any(|field| field.dtype == FieldDataType::Sketch(algorithm)), + _ => false, + }) + .collect() +} + +/// Whether some candidate of `group` survives Pass 2. +fn live(logical: &LogicalASAPMemoCandidates, group: MemoGroupId) -> bool { + (0..logical.memo.groups()[group.0 as usize].candidates.len()).any(|index_in_group| { + let candidate = CandidateForMemoGroupId { + group, + index_in_group, + }; + logical + .pruned + .iter() + .all(|pruned| pruned.candidate != candidate) + }) +} + +/// `quantile_over_time(0.9, m[5m])` twice, Q0 at rank error 0.01 and Q1 at +/// relative error 0.05: both share the quantile group and both the KLL +/// (rank) and DDSketch (relative) readouts survive, but Q0 reads only the +/// KLL readout and Q1 only the DDSketch one; each reads the other's +/// readout groups only exactly. +#[test] +fn each_query_reads_only_the_readouts_in_its_targets_metric() { + let query = "quantile_over_time(0.9, m[5m])"; + let logical = promql_pass2(&[ + (query, json!({"max_error": 0.01, "accuracy_metric": "rank"})), + ( + query, + json!({"max_error": 0.05, "accuracy_metric": "relative"}), + ), + ]); + let exact_only = &logical.exact_only_groups_of_query; + let kll = readout_groups(&logical, SketchAlgorithm::Kll); + let ddsketch = readout_groups(&logical, SketchAlgorithm::DDSketch); + assert!(!kll.is_empty() && !ddsketch.is_empty()); + for (group, rank_query_reads_it) in kll + .iter() + .map(|&group| (group, true)) + .chain(ddsketch.iter().map(|&group| (group, false))) + { + assert!(live(&logical, group), "G{} survives", group.0); + assert_eq!( + !exact_only[0].contains(&group), + rank_query_reads_it, + "G{}", + group.0 + ); + assert_eq!( + !exact_only[1].contains(&group), + !rank_query_reads_it, + "G{}", + group.0 + ); + } +} + +/// `count(quantile_over_time(0.9, m[5m]))` at relative error 0.05: the +/// count reads the quantiles through an exact count, whose contract +/// passes every metric, while Pass 1's Count-Min and UnivMon builds over +/// the quantiles (no contract) are dead. A dead candidate makes no group +/// exact-only: the quantile group and the DDSketch readout stay readable. +#[test] +fn a_dead_candidate_makes_no_group_exact_only() { + let logical = promql_pass2(&[( + "count(quantile_over_time(0.9, m[5m]))", + json!({"max_error": 0.05, "failure_probability": 0.01, "accuracy_metric": "relative"}), + )]); + let memo = &logical.memo; + let quantile = (0..memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .find(|group| { + matches!( + memo.groups()[group.0 as usize].expression, + Operator::NonASAP(NonASAPOp::Aggregate { ref measures, .. }) + if matches!(measures[..], [asap_types::ir::operator::AggIntent::Quantile { .. }]) + ) + }) + .expect("the quantile group"); + let ddsketch = readout_groups(&logical, SketchAlgorithm::DDSketch); + assert!(!ddsketch.is_empty()); + for group in ddsketch.into_iter().chain([quantile]) { + assert!(live(&logical, group), "G{} survives", group.0); + assert!( + !logical.exact_only_groups_of_query[0].contains(&group), + "G{} is exact-only", + group.0 + ); + } +} diff --git a/crates/integration-tests/tests/pass2_promql/rollup.rs b/crates/integration-tests/tests/pass2_promql/rollup.rs new file mode 100644 index 000000000..01b55413b --- /dev/null +++ b/crates/integration-tests/tests/pass2_promql/rollup.rs @@ -0,0 +1,133 @@ +//! PromQL → Pass 2's rollup rule, through production lowering: a coarse +//! cross-series aggregate is offered from a finer one over the same +//! per-series computation, although the two queries' scans list different +//! labels and an instant or range `TimeRange` (and a per-series function) +//! sits between each scan and its aggregate. +use crate::planning_common::promql_pass2; +use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; +use serde_json::json; + +/// Pass 2's memo of `queries`, each at a 1% error target. +fn pass2_memo(queries: &[&str]) -> Memo { + let queries: Vec<_> = queries + .iter() + .map(|&query| (query, json!({"max_error": 0.01}))) + .collect(); + promql_pass2(&queries).memo +} + +/// The root group of query `query`. +fn root(memo: &Memo, query: usize) -> MemoGroupId { + match memo.queries()[query].root { + QueryRoot::Operator(group) => group, + ref other => panic!("query {query} has no operator root: {other:?}"), + } +} + +/// The groups the rollup candidates of `group` read: the finer group +/// itself, or the finer group under the compensating `Filter`. +fn rolled_up_from(memo: &Memo, group: MemoGroupId) -> Vec { + memo.groups()[group.0 as usize] + .candidates + .iter() + .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|candidate| { + let [input] = candidate.expression.operator.children()[..] else { + panic!("a rollup reads one input: {:?}", candidate.expression); + }; + match &memo.groups()[input.0 as usize].expression { + Operator::NonASAP(NonASAPOp::Filter { child, .. }) => *child, + _ => *input, + } + }) + .collect() +} + +/// Every rollup candidate of the memo. +fn rollup_count(memo: &Memo) -> usize { + memo.groups() + .iter() + .flat_map(|group| &group.candidates) + .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup) + .count() +} + +#[test] +fn an_instant_sum_rolls_up_from_the_finer_sum_over_a_scan_listing_more_labels() { + let memo = pass2_memo(&["sum by (job) (m)", "sum by (job, instance) (m)"]); + assert_eq!(rolled_up_from(&memo, root(&memo, 0)), vec![root(&memo, 1)]); + assert!(rolled_up_from(&memo, root(&memo, 1)).is_empty()); +} + +/// A count rolls up as a sum of the finer counts, under the coarse +/// count's own column name (`count`, where the sum's would be `sum`). +#[test] +fn an_instant_count_rolls_up_from_the_finer_count() { + let memo = pass2_memo(&["count by (job) (m)", "count by (job, instance) (m)"]); + assert_eq!(rolled_up_from(&memo, root(&memo, 0)), vec![root(&memo, 1)]); +} + +#[test] +fn a_sum_of_rates_rolls_up_from_the_finer_sum_of_the_same_rates() { + let memo = pass2_memo(&[ + "sum by (job) (rate(m[5m]))", + "sum by (job, instance) (rate(m[5m]))", + ]); + assert_eq!(rolled_up_from(&memo, root(&memo, 0)), vec![root(&memo, 1)]); +} + +#[test] +fn a_label_matcher_becomes_a_compensating_filter_over_the_finer_sum() { + let memo = pass2_memo(&[ + r#"sum by (job) (m{instance="i1"})"#, + "sum by (job, instance) (m)", + ]); + let candidates = &memo.groups()[root(&memo, 0).0 as usize].candidates; + let rollup = candidates + .iter() + .find(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup) + .expect("a rollup candidate"); + let [filter] = rollup.expression.operator.children()[..] else { + panic!("a rollup reads one input"); + }; + let Operator::NonASAP(NonASAPOp::Filter { child, .. }) = + &memo.groups()[filter.0 as usize].expression + else { + panic!("the rollup reads a compensating filter"); + }; + assert_eq!(*child, root(&memo, 1)); +} + +#[test] +fn rates_over_different_ranges_do_not_roll_up() { + let memo = pass2_memo(&[ + "sum by (job) (rate(m[5m]))", + "sum by (job, instance) (rate(m[10m]))", + ]); + assert_eq!(rollup_count(&memo), 0); +} + +#[test] +fn different_per_series_functions_do_not_roll_up() { + let memo = pass2_memo(&[ + "sum by (job) (rate(m[5m]))", + "sum by (job, instance) (increase(m[5m]))", + ]); + assert_eq!(rollup_count(&memo), 0); +} + +#[test] +fn a_standard_deviation_does_not_roll_up() { + let memo = pass2_memo(&["stddev by (job) (m)", "stddev by (job, instance) (m)"]); + assert_eq!(rollup_count(&memo), 0); +} + +#[test] +fn a_finer_sum_narrowed_by_a_matcher_serves_no_wider_sum() { + let memo = pass2_memo(&[ + "sum by (job) (m)", + r#"sum by (job, instance) (m{instance="i1"})"#, + ]); + assert!(rolled_up_from(&memo, root(&memo, 0)).is_empty()); +} diff --git a/crates/integration-tests/tests/pass2_sql.rs b/crates/integration-tests/tests/pass2_sql.rs new file mode 100644 index 000000000..8b3f226fa --- /dev/null +++ b/crates/integration-tests/tests/pass2_sql.rs @@ -0,0 +1,61 @@ +//! SQL → Pass 2's readout requirements, through production lowering. +use crate::planning_common::sql_pass2; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId}; +use asap_types::summary_type::{SketchAlgorithm, SketchParams}; +use serde_json::json; + +/// `SELECT count(DISTINCT dst) FROM flows` at relative error 0.05 and +/// δ = 0.01 reads one distinct count per evaluation, so its KMV readout +/// keeps δ = 0.01 rather than splitting it over the 1000 groups of rows +/// scanned (1e-5), and a KMV build of k = 4323, within 0.05 at δ = 0.01, +/// survives. +#[test] +fn a_global_distinct_count_keeps_its_whole_failure_probability() { + let logical = sql_pass2(&[( + "SELECT count(DISTINCT dst) FROM flows", + json!({"max_error": 0.05, "failure_probability": 0.01, "accuracy_metric": "relative"}), + )]); + let memo = &logical.memo; + let mut kmv_readouts = 0; + let mut live_kmv_builds = Vec::new(); + for (group, memo_group) in memo.groups().iter().enumerate() { + for (index_in_group, candidate) in memo_group.candidates.iter().enumerate() { + let id = CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }; + let live = logical.pruned.iter().all(|pruned| pruned.candidate != id); + match &candidate.expression.operator { + Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) + if memo.groups()[summary_input.0 as usize] + .group_properties + .sketch_algorithm_of_state() + == Some(SketchAlgorithm::Kmv) => + { + let required = candidate.input_requirements[0] + .required_accuracy + .expect("a requirement"); + assert!( + (required.failure_probability - 0.01).abs() < 1e-12, + "{required:?}" + ); + kmv_readouts += 1; + } + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kmv), + .. + }) if live => { + if let Some(SketchParams::Kmv { k }) = candidate.logical_properties.parameters() + { + live_kmv_builds.push(*k); + } + } + _ => {} + } + } + } + assert!(kmv_readouts > 0); + assert!(live_kmv_builds.contains(&4323), "{live_kmv_builds:?}"); +} diff --git a/crates/integration-tests/tests/per_series_exact_execution.rs b/crates/integration-tests/tests/per_series_exact_execution.rs index 7ff4e94e0..d4510a74e 100644 --- a/crates/integration-tests/tests/per_series_exact_execution.rs +++ b/crates/integration-tests/tests/per_series_exact_execution.rs @@ -1,5 +1,4 @@ -//! Exact per-series range reductions over raw data samples execute natively -//! (issue #770): exact `quantile/sum/avg/min/max/count_over_time` reduces +//! Exact per-series range reductions over raw data samples execute natively: exact `quantile/sum/avg/min/max/count_over_time` reduces //! each series' samples in the window with a per-entity `Aggregate`, which //! the executor compiles and runs. The samples are bound at the plan's //! `Scan`, as a runtime binds them, and its `TimeRange` keeps the window. @@ -94,7 +93,7 @@ fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap { /// at its `Scan` of `x`: each answered series' `instance` and value. fn execute(query: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { let plan = per_entity_plan(query); - let dag = asap_types::test_support::physical_asap_dag_at_query_time(&plan).unwrap(); + let dag = asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&plan).unwrap(); let node = |matches: fn(&PhysicalASAPOperatorPayload) -> bool| { let found: Vec<_> = dag .nodes diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 84181645c..d29a3ff36 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -55,7 +55,7 @@ pub fn execute_raw_rows_at( use asap_executor::physical_planner::compile; use asap_types::ir::NonASAPOp; use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; - let wire = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); + let wire = asap_types::test_support::forwarders::physical_asap_dag_at_query_time(root).unwrap(); let (source, schema) = wire .nodes .iter() diff --git a/crates/integration-tests/tests/plan_sharing.rs b/crates/integration-tests/tests/plan_sharing.rs new file mode 100644 index 000000000..2a11ee899 --- /dev/null +++ b/crates/integration-tests/tests/plan_sharing.rs @@ -0,0 +1,121 @@ +//! Workloads whose queries share computation, planned through the production +//! stages: adding a query that can read an existing result must not cost +//! more than planning the first query alone. +use crate::planning_common::{cost, fuzz_workload}; +use serde_json::json; + +/// Deriving the coarse sum costs only a sum over the 150 fine groups. +/// Stage 2 offers that plan; Stage 3 selects a `OneEvaluation` build whose +/// not-materialized sibling would avoid a second scan. +#[test] +#[ignore = "Stage 3 misses the cheaper plan Stage 2 offers (#1142)"] +fn exact_coarse_sum_reuses_fine_sum() { + let related = fuzz_workload("avg_sum_by_instance_region_and_instance", false); + let mut base = related.clone(); + base.queries.truncate(1); + let base_cost = cost(&base); + let related_cost = cost(&related); + assert!( + related_cost <= base_cost + 0.00004, + "{related_cost} vs {base_cost}" + ); +} + +/// The 1m sum reads the newest tumbling window of the existing 2m chain. +#[test] +fn nested_exact_sum_reuses_newest_tumbling_windows() { + let related = fuzz_workload("nested_exact_sum", false); + let mut base = related.clone(); + base.queries.truncate(1); + let base_cost = cost(&base); + let related_cost = cost(&related); + assert!( + related_cost <= base_cost * 1.001, + "{related_cost} vs {base_cost}" + ); +} + +/// The slower copy reads the same maintained sum. +#[test] +fn slower_repeat_reuses_faster_sum() { + let related = fuzz_workload("slower_repeat_exact_sum", false); + let mut base = related.clone(); + base.queries.truncate(1); + let base_cost = cost(&base); + let related_cost = cost(&related); + assert!( + related_cost <= base_cost * 1.001, + "{related_cost} vs {base_cost}" + ); +} + +/// The filtered query avoids its independent scan and aggregation. +#[test] +fn grouped_filtered_quantile_plans() { + assert!( + cost(&fuzz_workload("percentile_filtered_and_by_city", true)) < 0.25, + "filtered query must reuse the grouped result" + ); +} + +/// Duplicate exact TopK queries share their result and inner window work. +#[test] +fn duplicate_exact_topk_shares_work() { + let mut w = fuzz_workload("nested_exact_sum", false); + for query in &mut w.queries { + query.entry.query.0 = "topk(3, sum_over_time(m_gauge[2m]))".into(); + } + let mut base = w.clone(); + base.queries.truncate(1); + let single = cost(&base); + let duplicate = cost(&w); + assert!(duplicate <= single * 1.001, "{duplicate} vs {single}"); +} + +/// SQL variants of the nested and slower-repeat cases: nested windows and slower queries reuse one sum. +#[test] +fn sql_sum_shares_nested_windows_and_slower_readers() { + let mut w = fuzz_workload("identical_percentiles", false); + w.queries.truncate(1); + w.queries[0].entry.query.0 = + "SELECT SUM(value) FROM metrics WHERE ts > now() - INTERVAL '5' MINUTE".into(); + w.queries[0].entry.demand = serde_json::from_value(json!({"fixed_interval":60000})).unwrap(); + let single = cost(&w); + let mut nested = w.clone(); + let mut shorter = w.queries[0].clone(); + shorter.entry.query.0 = shorter.entry.query.0.replace("'5'", "'1'"); + nested.queries.push(shorter); + let nested_cost = cost(&nested); + assert!(nested_cost <= single * 1.001, "{nested_cost} vs {single}"); + let mut slower = w.queries[0].clone(); + slower.entry.demand = serde_json::from_value(json!({"fixed_interval":120000})).unwrap(); + w.queries.push(slower); + let repeated_cost = cost(&w); + assert!( + repeated_cost <= single * 1.001, + "{repeated_cost} vs {single}" + ); +} + +/// Two identical exact percentile queries pay for their result once. +#[test] +fn identical_exact_percentiles_share_work() { + let related = fuzz_workload("identical_percentiles", false); + let mut base = related.clone(); + base.queries.truncate(1); + let base_cost = cost(&base); + let related_cost = cost(&related); + assert!( + related_cost <= base_cost * 1.001, + "{related_cost} vs {base_cost}" + ); +} + +/// Different ranks share the source window but still compute their own values. +#[test] +fn different_exact_percentiles_share_the_input_window() { + let mut related = fuzz_workload("identical_percentiles", false); + related.queries[1].entry.query.0 = related.queries[1].entry.query.0.replace("0.99", "0.5"); + let related_cost = cost(&related); + assert!(related_cost <= 0.0883, "{related_cost}"); +} diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/923.json b/crates/integration-tests/tests/plan_sharing_fixtures/avg_sum_by_instance_region_and_instance.json similarity index 100% rename from crates/integration-tests/tests/sharing_issue_fixtures/923.json rename to crates/integration-tests/tests/plan_sharing_fixtures/avg_sum_by_instance_region_and_instance.json diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/922.json b/crates/integration-tests/tests/plan_sharing_fixtures/identical_percentiles.json similarity index 100% rename from crates/integration-tests/tests/sharing_issue_fixtures/922.json rename to crates/integration-tests/tests/plan_sharing_fixtures/identical_percentiles.json diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/924.json b/crates/integration-tests/tests/plan_sharing_fixtures/nested_exact_sum.json similarity index 100% rename from crates/integration-tests/tests/sharing_issue_fixtures/924.json rename to crates/integration-tests/tests/plan_sharing_fixtures/nested_exact_sum.json diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/902.json b/crates/integration-tests/tests/plan_sharing_fixtures/percentile_filtered_and_by_city.json similarity index 100% rename from crates/integration-tests/tests/sharing_issue_fixtures/902.json rename to crates/integration-tests/tests/plan_sharing_fixtures/percentile_filtered_and_by_city.json diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/925.json b/crates/integration-tests/tests/plan_sharing_fixtures/slower_repeat_exact_sum.json similarity index 100% rename from crates/integration-tests/tests/sharing_issue_fixtures/925.json rename to crates/integration-tests/tests/plan_sharing_fixtures/slower_repeat_exact_sum.json diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index ca8b332db..a3bfa78ad 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -1,4 +1,4 @@ -//! Shared helpers for the #509 Examples 1–4 acceptance tests +//! Shared helpers for the Examples 1–4 acceptance tests //! (`planner_layering_example{1,2,3,4}.rs`). Specs: //! `docs/design_docs/proposals/planner-layering-example{1,2,3,4}-acceptance.md`. //! @@ -307,13 +307,13 @@ pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTre .collect() } -/// How many times each node is computed per evaluation of its consumers (#509; the +/// How many times each node is computed per evaluation of its consumers (the /// rule of plan selection's private `computations`): 1 for a materialized /// node; for a not-materialized one, the number of consumers that reach /// it. A consumer is a query (operator or scalar root) or a materialized /// node, whose walk starts at its inputs; a walk stops at materialized /// nodes and counts each node once. The consumers materialized at ingestion time of one -/// refresh window share one ingestion run and count once (#1151). +/// refresh window share one ingestion run and count once. pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { let materialized_nodes: BTreeSet<_> = dag .nodes @@ -393,7 +393,7 @@ pub enum Materializing { } /// Whether physical ASAP-aware optimization materializes `node`'s output, and at -/// which timing (#667). +/// which timing. pub fn materializing(p: &Physical, node: NodeId) -> Materializing { let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); match (n.output_state.timing, n.lifetime) { @@ -410,7 +410,7 @@ pub fn runs_at_ingestion(p: &Physical, node: NodeId) -> bool { // ── Pending adapters ───────────────────────────────────────────────────── -/// The window summary a summary build node maintains (#509 Pass 2, +/// The window summary a summary build node maintains (Pass 2's /// window-composition rule). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum WindowForm { @@ -426,8 +426,7 @@ pub enum WindowForm { /// The window summary of a build: a `SummaryTumblingWindowChain` keeps back-to-back tumbling windows of /// its tumbling-window width (tumbling); a `SummaryAgg` is rebuilt from its query -/// window. Sliding windows and Exponential Histograms are not planned yet -/// (#628). +/// window. Sliding windows and Exponential Histograms are not planned yet. pub fn window_form(dag: &impl ExportedDag, build: NodeId) -> WindowForm { match dag.payload(build) { Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { diff --git a/crates/integration-tests/tests/planning_common/mod.rs b/crates/integration-tests/tests/planning_common/mod.rs new file mode 100644 index 000000000..809bcbc80 --- /dev/null +++ b/crates/integration-tests/tests/planning_common/mod.rs @@ -0,0 +1,147 @@ +//! Helpers shared by the planning tests that load a workload fixture, plan it +//! through the production stages, and compare costs, or that run PromQL or +//! SQL workloads through Pass 2. +use asap_cost::CostInputs; +use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; +use asap_devtools::planning_stages::{ + lower_promql_roots, lower_sql_roots, run_search_over, run_stages_until, sql_catalog, RunUntil, +}; +use asap_planner::{PlanSearchSettings, PlanningModels}; +use asap_types::stages::LogicalASAPMemoCandidates; +use asap_types::workload::PlanningWorkload; +use serde_json::{json, Value}; +use std::path::Path; + +/// The workload in `tests/plan_sharing_fixtures/.json` over the +/// `metrics(ts, city, host, value)` table, whose columns after `ts` are +/// nullable when `nullable_columns` is set. +pub fn fuzz_workload(fixture: &str, nullable_columns: bool) -> FuzzWorkload { + let mut w = read_seed_file( + &Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/plan_sharing_fixtures") + .join(format!("{fixture}.json")), + ) + .unwrap(); + w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ + {"name":"ts","type":"timestamp","nullable":false}, + {"name":"city","type":"utf8","nullable":nullable_columns}, + {"name":"host","type":"utf8","nullable":nullable_columns}, + {"name":"value","type":"float64","nullable":nullable_columns} + ]})]; + w +} + +/// `w` with its queries replaced by `queries`, each keeping the demand and +/// requirements of the workload's first query. +pub fn with_queries(mut w: FuzzWorkload, queries: &[String]) -> FuzzWorkload { + let template = w.queries[0].clone(); + w.queries = queries + .iter() + .map(|query| { + let mut entry = template.clone(); + entry.entry.query.0 = query.clone(); + entry + }) + .collect(); + w +} + +/// The selected plan's cost per second for `w`. +pub fn cost(w: &FuzzWorkload) -> f64 { + let roots = Planner::new(FuzzSettings::default()).lower(w).unwrap(); + let planning = w.planning_workload(); + let entries: Vec<_> = planning.query_workload.entries().collect(); + let settings = CostInputs::default(); + let deployment = w.deployment(); + let models = PlanningModels::builtin() + .with_cost_inputs(&settings) + .with_deployment_capabilities(&deployment); + let run = run_search_over( + roots, + &entries, + &w.data, + models, + PlanSearchSettings::default(), + ) + .unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); + println!("cost={value}"); + value +} + +/// Pass 2's output for PromQL `queries`, each with its accuracy target, +/// repeating every minute over the gauge `m` and the counter `c`, 1000 +/// series each. +pub fn promql_pass2(queries: &[(&str, Value)]) -> LogicalASAPMemoCandidates { + pass2_of( + json!("promql"), + queries, + json!({ + "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, + "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + }), + None, + ) +} + +/// Pass 2's output for SQL `queries` over the table +/// `flows(ts, src, dst, bytes)` of 1000 groups of rows, each with its +/// accuracy target, repeating every minute. +pub fn sql_pass2(queries: &[(&str, Value)]) -> LogicalASAPMemoCandidates { + let flows = json!({"name": "flows", "time_index": 0, "columns": [ + {"name": "ts", "type": "timestamp"}, + {"name": "src", "type": "utf8"}, + {"name": "dst", "type": "utf8"}, + {"name": "bytes", "type": "float64"}, + ]}); + pass2_of( + json!({"sql": "datafusion_sql"}), + queries, + json!({"flows": {"series_count": 1000}}), + Some(&[flows.to_string()]), + ) +} + +/// Pass 2's output for `queries` in `language` over `sources`, through +/// production lowering (SQL over the `tables` catalog). +fn pass2_of( + language: Value, + queries: &[(&str, Value)], + sources: Value, + tables: Option<&[String]>, +) -> LogicalASAPMemoCandidates { + let workload: PlanningWorkload = serde_json::from_value(json!({ + "query_workload": { + "language": language, + "repeating_queries": queries.iter().map(|(query, accuracy)| json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}, + "requirements": {"accuracy": accuracy}, + "predictability": {"predictable": {"known_at": null}}, + })).collect::>(), + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": sources, + }, + })) + .unwrap(); + workload.validate().unwrap(); + let roots = match tables { + Some(tables) => lower_sql_roots(&workload, &sql_catalog(tables).unwrap().0).unwrap(), + None => lower_promql_roots(&workload).unwrap(), + }; + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_stages_until( + roots, + &entries, + &data, + PlanningModels::builtin(), + RunUntil::Pass2, + ) + .unwrap(); + run.run.logical_asap_candidates.unwrap() +} diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 2f0f0c7e2..704ee1c23 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -431,7 +431,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // is an aggregation operator with no `by(...)`: a genuine full // reduction, one output row — not to be confused with the inner rate's // per-entity grouping below, even though both once collapsed to the - // same empty `by: []` (issue #163). + // same empty `by: []`. let Some(ASAPOp::SummaryAgg { child, summary_type: family, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/886.json b/crates/integration-tests/tests/sharing_issue_fixtures/886.json deleted file mode 100644 index 259fafd9d..000000000 --- a/crates/integration-tests/tests/sharing_issue_fixtures/886.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "query_workload": { - "language": { - "sql": "datafusion_sql" - }, - "query_batch": null, - "repeating_queries": [ - { - "query": "SELECT host, COUNT(*) AS c FROM metrics WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY host ORDER BY c DESC LIMIT 5", - "demand": { - "fixed_interval": 10000 - }, - "requirements": { - "accuracy": { - "max_error": 0.01, - "failure_probability": 0.001 - } - }, - "predictability": { - "predictable": { - "known_at": null - } - } - } - ] - }, - "data_workload": { - "arrival": { - "ingesting": { - "rows_per_second": 666.67 - } - }, - "sources": { - "metrics": { - "series_count": 100 - } - } - } -} diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs deleted file mode 100644 index 8d896009f..000000000 --- a/crates/integration-tests/tests/sharing_issue_regressions.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! Original issue workloads against the production candidate-connection planner. -use asap_cost::CostInputs; -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use asap_devtools::planning_stages::run_search_over; - -use asap_planner::{PlanSearchSettings, PlanningModels}; - -use serde_json::json; -use std::path::Path; - -fn workload(issue: u32) -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")) - .join(format!("tests/sharing_issue_fixtures/{issue}.json")), - ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":false}, - {"name":"host","type":"utf8","nullable":false}, - {"name":"value","type":"float64","nullable":false} - ]})]; - if issue == 902 || issue == 886 { - for field in w.tables[0]["columns"] - .as_array_mut() - .unwrap() - .iter_mut() - .skip(1) - { - field["nullable"] = json!(true); - } - } - w -} - -fn cost(w: &FuzzWorkload) -> f64 { - let roots = Planner::new(FuzzSettings::default()).lower(w).unwrap(); - let planning = w.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - let settings = CostInputs::default(); - let deployment = w.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment); - let run = run_search_over( - roots, - &entries, - &w.data, - models, - PlanSearchSettings::default(), - ) - .unwrap(); - let (plan, _) = run.run.selection.as_ref().unwrap(); - let value = - asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); - println!("cost={value}"); - value -} - -/// #923: deriving the coarse sum costs only a sum over the 150 fine groups. -/// Stage 2 offers that plan; Stage 3 selects a `OneEvaluation` build whose -/// not-materialized sibling would avoid a second scan (#1142). -#[test] -#[ignore = "Stage 3 misses the cheaper plan Stage 2 offers (#1142)"] -fn exact_coarse_sum_reuses_fine_sum() { - let related = workload(923); - let mut base = related.clone(); - base.queries.truncate(1); - let base_cost = cost(&base); - let related_cost = cost(&related); - assert!( - related_cost <= base_cost + 0.00004, - "{related_cost} vs {base_cost}" - ); -} - -/// #924: the 1m sum reads the newest tumbling window of the existing 2m chain. -#[test] -fn nested_exact_sum_reuses_newest_tumbling_windows() { - let related = workload(924); - let mut base = related.clone(); - base.queries.truncate(1); - let base_cost = cost(&base); - let related_cost = cost(&related); - assert!( - related_cost <= base_cost * 1.001, - "{related_cost} vs {base_cost}" - ); -} - -/// #925: the slower copy reads the same maintained sum. -#[test] -fn slower_repeat_reuses_faster_sum() { - let related = workload(925); - let mut base = related.clone(); - base.queries.truncate(1); - let base_cost = cost(&base); - let related_cost = cost(&related); - assert!( - related_cost <= base_cost * 1.001, - "{related_cost} vs {base_cost}" - ); -} - -/// #902: the filtered query avoids its independent scan and aggregation. -#[test] -fn grouped_filtered_quantile_plans() { - assert!( - cost(&workload(902)) < 0.25, - "filtered query must reuse the grouped result" - ); -} - -/// #922: duplicate exact TopK queries share their result and inner window work. -#[test] -fn duplicate_exact_topk_shares_work() { - let mut w = workload(924); - for query in &mut w.queries { - query.entry.query.0 = "topk(3, sum_over_time(m_gauge[2m]))".into(); - } - let mut base = w.clone(); - base.queries.truncate(1); - let single = cost(&base); - let duplicate = cost(&w); - assert!(duplicate <= single * 1.001, "{duplicate} vs {single}"); -} - -/// #924/#925 SQL variants: nested windows and slower queries reuse one sum. -#[test] -fn sql_sum_shares_nested_windows_and_slower_readers() { - let mut w = workload(922); - w.queries.truncate(1); - w.queries[0].entry.query.0 = - "SELECT SUM(value) FROM metrics WHERE ts > now() - INTERVAL '5' MINUTE".into(); - w.queries[0].entry.demand = serde_json::from_value(json!({"fixed_interval":60000})).unwrap(); - let single = cost(&w); - let mut nested = w.clone(); - let mut shorter = w.queries[0].clone(); - shorter.entry.query.0 = shorter.entry.query.0.replace("'5'", "'1'"); - nested.queries.push(shorter); - let nested_cost = cost(&nested); - assert!(nested_cost <= single * 1.001, "{nested_cost} vs {single}"); - let mut slower = w.queries[0].clone(); - slower.entry.demand = serde_json::from_value(json!({"fixed_interval":120000})).unwrap(); - w.queries.push(slower); - let repeated_cost = cost(&w); - assert!( - repeated_cost <= single * 1.001, - "{repeated_cost} vs {single}" - ); -} diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 48e6fb721..ac1f7cfd0 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -1,4 +1,4 @@ -//! End-to-end SQL query-string → post-ASAP IR pin (issue #191). +//! End-to-end SQL query-string → post-ASAP IR pin. //! //! The SQL counterpart of `promql_to_post_asap.rs`: drives SQL text — //! `lower_sql` (text → non-ASAP `OperatorNode` tree) → the planner stages' diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index df5f6669f..fd7b1fdb6 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -44,7 +44,7 @@ use crate::{e2e_plan, PlanError, UserInput}; /// The version of the plan-file format [`InstallablePlan`] serializes to: /// the DAG, its nodes' fields and the query texts. -pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 14; +pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 15; /// The selected plan, ready to install and execute: what /// [`plan_installable`] returns and a runtime writes as its plan file. diff --git a/crates/stage1-pass1-per-computation-candidates/Cargo.toml b/crates/stage1-pass1-per-computation-candidates/Cargo.toml index 0c8e36180..39cc1fc02 100644 --- a/crates/stage1-pass1-per-computation-candidates/Cargo.toml +++ b/crates/stage1-pass1-per-computation-candidates/Cargo.toml @@ -3,14 +3,12 @@ name = "asap-stage1-pass1-per-computation-candidates" version = "0.1.0" edition = "2021" -# #509 Stage 1, Pass 1: the initial candidates of each memo group (exact, -# exact accumulator, summaries, fused builds), with the shared analytical -# accuracy model (`asap-accuracy`). Reads Stage 0's memo through +# Stage 1, Pass 1: the initial candidates of each memo group (exact, +# exact accumulator, summaries, fused builds). Reads Stage 0's memo through # `asap_types::stages`; depends on no other stage crate and on no # query-language frontend. [dependencies] asap-types = { path = "../types" } -asap-accuracy = { path = "../accuracy" } thiserror = "2" [dev-dependencies] diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/having.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/having.rs new file mode 100644 index 000000000..3623943f1 --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/having.rs @@ -0,0 +1,38 @@ +//! Where an aggregate's `HAVING` sits in the memo: in its own `having` or in +//! a consumer `Filter` group directly above it. + +use asap_types::ir::{NonASAPOp, Operator, Predicate}; +use asap_types::stages::{Memo, MemoGroupId}; + +use crate::expression; + +/// The consumer `Filter` groups directly above `aggregate_group` (its +/// `HAVING` as the SQL frontend lowers it), with their predicates. A +/// `Filter` that reads the aggregate from a scalar expression stays a plain +/// consumer. +pub(super) fn having_filter_groups( + memo: &Memo, + aggregate_group: MemoGroupId, +) -> impl Iterator)> { + memo.groups()[aggregate_group.0 as usize] + .parent_groups() + .iter() + .filter_map(move |&parent| match expression(memo, parent) { + Operator::NonASAP(NonASAPOp::Filter { child, pred }) if *child == aggregate_group => { + Some((parent, pred)) + } + _ => None, + }) +} + +/// Whether `aggregate_group` has a `HAVING`: its own `having` or a consumer +/// `Filter` group directly above it ([`having_filter_groups`]). +pub(crate) fn has_having(memo: &Memo, aggregate_group: MemoGroupId) -> bool { + matches!( + expression(memo, aggregate_group), + Operator::NonASAP(NonASAPOp::Aggregate { + having: Some(_), + .. + }) + ) || having_filter_groups(memo, aggregate_group).next().is_some() +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs index a8e5e2689..9b8178fa4 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs @@ -13,16 +13,12 @@ use asap_types::ir::{ ProjectItem, ScalarExpr, }; use asap_types::stages::{Memo, MemoGroupId}; -use asap_types::summary_type::{ExactKind, SummaryInputExpr}; - -use asap_accuracy::sizing::sketch_statistic; +use asap_types::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr}; use super::pending::{add_candidate, commit_pending, Pending}; use super::SingleMeasure; use crate::computation_targets::HowToComputeAggregateMeasure; -use crate::summary_realization::{ - measure_input, summary_update, with_raw_data_samples_requirement, -}; +use crate::summary_realization::{measure_input, summary_update, with_raw_data_samples_of}; use crate::{schema, PerComputationCandidateError}; /// The operator that computes `measure` by `way` for @@ -57,7 +53,7 @@ pub(super) fn measure_output( measure.unfiltered_input, measure.filter, )?; - let update = with_raw_data_samples_requirement( + let update = with_raw_data_samples_of( summary_update( measure.intent, &summary_type, @@ -180,9 +176,9 @@ pub(super) fn summary_readout( reduction: &Reduction, result_group: MemoGroupId, ) -> Result, PerComputationCandidateError> { - let query = sketch_statistic(intent).ok_or(PerComputationCandidateError::Unsupported( - "sketch evaluation", - ))?; + let query = SketchStatistic::for_intent(intent).ok_or( + PerComputationCandidateError::Unsupported("sketch evaluation"), + )?; let state = pending.push(memo, build)?; let estimate = Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: state, diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs index 30992f0e8..a863c7551 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs @@ -43,6 +43,7 @@ //! every check that can find it unsupported has passed, so an unsupported //! way adds nothing to the memo. +pub(crate) mod having; mod measure_output; mod pending; @@ -55,11 +56,12 @@ use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind, Predicate, use asap_types::stages::{raw_data_samples_of_metric_of_group, Memo, MemoGroupId}; use crate::computation_targets::{ - having_filter_groups, nested_distributive_input, whole_expression_input, ComputationTarget, + nested_distributive_input, whole_expression_input, ComputationTarget, HowToComputeAggregateMeasure, }; -use crate::summary_realization::{measure_input, with_raw_data_samples_requirement}; +use crate::summary_realization::{measure_input, with_raw_data_samples_of}; use crate::{expression, PerComputationCandidateError}; +use having::having_filter_groups; use measure_output::{ accumulator_build, add_output_candidate, join_measures, measure_output, summary_readout, }; @@ -443,7 +445,7 @@ fn add_fused_candidate( )?; let build = Operator::ASAP(ASAPOp::SummaryAgg { child: rows, - input: with_raw_data_samples_requirement(update, measure.raw_data_samples_of), + input: with_raw_data_samples_of(update, measure.raw_data_samples_of), summary_type: FieldDataType::Sketch(*algorithm), reduction: partitions, filter: None, diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs index 44fd9d99e..e375abdb2 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs @@ -4,14 +4,13 @@ //! check that can find the way unsupported has passed, so an unsupported //! way adds nothing. -use asap_types::ir::schema::FieldDataType; use asap_types::ir::schema::Schema; use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ - CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, - MemoGroupId, OriginOfCandidateForMemoGroup, + listed_build_parameters, CandidateForMemoGroup, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, }; -use asap_types::summary_type::{parameter_list, SketchParams}; +use asap_types::summary_type::SketchParams; use crate::{schema, PerComputationCandidateError}; @@ -135,18 +134,7 @@ pub(crate) fn add_state_group( fused_groups: &[MemoGroupId], ) -> Result { let group = memo.find_or_add_group(build.clone())?; - let configurations: Vec> = match &build { - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(algorithm), - .. - }) => parameter_list(*algorithm) - .iter() - .cloned() - .map(Some) - .collect(), - _ => vec![None], - }; - for parameters in configurations { + for parameters in listed_build_parameters(&build) { add_candidate( memo, group, @@ -168,19 +156,14 @@ pub(crate) fn add_candidate( parameters: Option, fused_groups: Vec, ) -> Result<(), PerComputationCandidateError> { - let expression = CandidateExpression { operator }; - let logical_properties = LogicalProperties::new(&expression, parameters)?; - let input_requirements = - vec![LogicalRequirements::default(); expression.operator.children().len()]; memo.add_candidate( group, - CandidateForMemoGroup { - expression, - logical_properties, - input_requirements, + CandidateForMemoGroup::new( + operator, + parameters, + OriginOfCandidateForMemoGroup::Pass1, fused_groups, - origin: OriginOfCandidateForMemoGroup::Pass1, - }, + ), )?; Ok(()) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index 91839069c..9e2b5751a 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -20,15 +20,15 @@ use asap_types::ir::operator::non_asap::any_measure_filtered; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::{ColumnId, Schema}; -use asap_types::ir::{NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr}; +use asap_types::ir::{NonASAPOp, Operator, OperatorResultKind, ScalarExpr}; use asap_types::stages::{Memo, MemoGroupId}; use asap_types::summary_type::{ - ExactKind, SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, + ExactKind, SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryUpdate, }; -use asap_accuracy::sizing::sketch_statistic; use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; +use crate::candidates::having::has_having; use crate::summary_realization::{realize_keyed_additive_summary_input, summary_candidates}; use crate::{expression, schema, PerComputationCandidateError}; @@ -97,9 +97,7 @@ pub(crate) struct FusedComputation { /// one; plain exact execution is not listed (it is the group's own /// expression). A quantile outside [0, 1] gets no summary: its answer is /// -Inf or +Inf. -pub(crate) fn computation_candidates_for_intent( - intent: &AggIntent, -) -> Vec { +pub(crate) fn ways_to_compute_intent(intent: &AggIntent) -> Vec { let states = match intent { AggIntent::Count => vec![ExactKind::Count], AggIntent::Sum { .. } => vec![ExactKind::Sum], @@ -117,7 +115,7 @@ pub(crate) fn computation_candidates_for_intent( } let quantile_outside_unit_interval = matches!(intent, AggIntent::Quantile { q, .. } if !(0.0..=1.0).contains(q)); - if sketch_statistic(intent).is_some() && !quantile_outside_unit_interval { + if SketchStatistic::for_intent(intent).is_some() && !quantile_outside_unit_interval { choices.extend( summary_candidates(intent) .into_iter() @@ -139,37 +137,6 @@ fn nullable_avg_input(input: &Schema, intent: &AggIntent) -> bool { .is_none_or(|field| field.nullable) } -/// The consumer `Filter` groups directly above `aggregate_group` (its -/// `HAVING` as the SQL frontend lowers it), with their predicates. A -/// `Filter` that reads the aggregate from a scalar expression stays a plain -/// consumer. -pub(crate) fn having_filter_groups( - memo: &Memo, - aggregate_group: MemoGroupId, -) -> impl Iterator)> { - memo.groups()[aggregate_group.0 as usize] - .parent_groups() - .iter() - .filter_map(move |&parent| match expression(memo, parent) { - Operator::NonASAP(NonASAPOp::Filter { child, pred }) if *child == aggregate_group => { - Some((parent, pred)) - } - _ => None, - }) -} - -/// Whether `aggregate_group` has a `HAVING`: its own `having` or a consumer -/// `Filter` group directly above it ([`having_filter_groups`]). -fn has_having(memo: &Memo, aggregate_group: MemoGroupId) -> bool { - matches!( - expression(memo, aggregate_group), - Operator::NonASAP(NonASAPOp::Aggregate { - having: Some(_), - .. - }) - ) || having_filter_groups(memo, aggregate_group).next().is_some() -} - /// Discover one target per measure of each eligible aggregate group of /// `memo`, nested aggregates and those read by scalar roots included: /// Pass 1 walks the groups, so an aggregate several queries read is one @@ -202,7 +169,7 @@ pub(crate) fn computation_targets( }; let aggregate_group = MemoGroupId(index as u32); for (measure, intent) in measures.iter().enumerate() { - let mut choices = computation_candidates_for_intent(intent); + let mut choices = ways_to_compute_intent(intent); if choices.is_empty() { continue; } @@ -251,7 +218,8 @@ fn without_unsupported_summaries( else { continue; }; - let Some(readout) = sketch_statistic(&measures[target.measure_index]).map(|s| s.kind()) + let Some(readout) = + SketchStatistic::for_intent(&measures[target.measure_index]).map(|s| s.kind()) else { continue; }; diff --git a/crates/stage1-pass1-per-computation-candidates/src/lib.rs b/crates/stage1-pass1-per-computation-candidates/src/lib.rs index 0effe69fe..9a5ffbe82 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/lib.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/lib.rs @@ -7,8 +7,7 @@ //! group; Pass 2 extends that memo. //! //! It depends only on `asap-types` (its input and output memo are in -//! `asap_types::stages`) and the shared accuracy model (`asap-accuracy`, -//! for the statistic each intent reads). It depends on no other stage +//! `asap_types::stages`). It depends on no other stage //! crate, and the planner facade calls it after Stage 0. It depends on no //! query-language frontend either; its tests build their input memo through //! `Memo::find_or_add_group` and `Memo::add_query`. @@ -37,9 +36,7 @@ mod tests; use asap_types::deployment_capabilities::SupportedSummary; use asap_types::ir::schema::Schema; use asap_types::ir::{Operator, SchemaDerivationError}; -use asap_types::stages::{ - CandidateError, LogicalPropertiesError, Memo, MemoGroupError, MemoGroupId, -}; +use asap_types::stages::{CandidateError, Memo, MemoGroupError, MemoGroupId}; use thiserror::Error; use candidates::add_pass1_candidates; @@ -62,10 +59,6 @@ pub enum PerComputationCandidateError { /// its group's result: a bug in the rule that built it. #[error(transparent)] Candidate(#[from] CandidateError), - /// A candidate Pass 1 built has logical properties that cannot go - /// together: a bug in the rule that built it. - #[error(transparent)] - LogicalProperties(#[from] LogicalPropertiesError), #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] Unsupported(&'static str), } diff --git a/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs b/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs index 4d86fa54a..94cb6678d 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs @@ -15,12 +15,10 @@ use asap_types::stages::{ ranked_column_of_group, raw_data_samples_of_metric_of_group, Memo, MemoGroupId, }; use asap_types::summary_type::{ - EntityIdentity, NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr, + EntityIdentity, NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, }; -use asap_accuracy::sizing::sketch_statistic; - use crate::computation_targets::nested_aggregates; use crate::{expression, schema, PerComputationCandidateError}; @@ -45,7 +43,7 @@ const ALGORITHM_PREFERENCE: &[SketchAlgorithm] = &[ /// count. Empty for an intent no sketch /// statistic serves. pub(crate) fn summary_candidates(intent: &AggIntent) -> Vec { - let Some(statistic) = sketch_statistic(intent) else { + let Some(statistic) = SketchStatistic::for_intent(intent) else { return Vec::new(); }; // A distinct-tuple count hashes the whole tuple as one item @@ -286,7 +284,7 @@ fn summed_per_series_samples( /// `update` with the open requirement that its weights are samples of /// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap /// update whose weights are not otherwise proven. -pub(crate) fn with_raw_data_samples_requirement( +pub(crate) fn with_raw_data_samples_of( mut update: SummaryUpdate, metric: Option<&str>, ) -> SummaryUpdate { diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs index bb0ec22e0..3102f543a 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -1,38 +1,13 @@ -//! Fixtures this crate's own tests share. The memo builders add Stage 0's -//! groups through `Memo::find_or_add_group` and `Memo::add_query`, as -//! Stage 0 does, so no test calls another stage crate. The expressions -//! come from `asap_types::test_support::memo_fixtures`; its scans list Utf8 -//! labels. +//! Fixtures this crate's own tests share, beside the memo builders of +//! `asap_types::test_support::memo_builders` (`group`, `scan_group`, +//! `query`), so no test calls another stage crate. use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::series_identity::with_promql_series_identity; -use asap_types::ir::schema::DataType; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot}; -use asap_types::stages::{Memo, MemoGroupId, MemoQuery}; -use asap_types::test_support::memo_fixtures::{aggregate_with, scan}; - -/// The group of `operator`, added as Stage 0 adds it. -pub fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { - memo.find_or_add_group(operator).unwrap() -} - -/// The group of a `TimeSeries(metric)` scan listing `labels`, its series -/// identity realized as the planner realizes it before Stage 0 -/// (`with_promql_series_identity`): a closed schema whose last column is -/// `$promql_series_identity`, keyed by it and the timestamp. -pub fn scan_group(memo: &mut Memo, metric: &str, labels: &[&str]) -> MemoGroupId { - let labels: Vec<(&str, DataType)> = labels.iter().map(|l| (*l, DataType::Utf8)).collect(); - let mut operator = scan(metric, &labels, vec![]); - let open = - OperatorNode::new_shared(operator.map_children(|_| unreachable!("a scan has no input"))) - .unwrap(); - let realized = with_promql_series_identity(&open).unwrap().schema.clone(); - if let Operator::NonASAP(NonASAPOp::Scan { schema, .. }) = &mut operator { - *schema = realized; - } - group(memo, operator) -} +use asap_types::ir::Predicate; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::test_support::memo_builders::group; +use asap_types::test_support::memo_fixtures::aggregate_with; /// The group of an `Aggregate` of `measures` (unnamed, unfiltered) over /// `child`, grouped by `reduction`, with `having`. @@ -46,12 +21,3 @@ pub fn aggregate_group( let measures = measures.into_iter().map(|m| (m, None)).collect(); group(memo, aggregate_with(reduction, measures, having, child)) } - -/// Adds a query whose root is group `root`. -pub fn query(memo: &mut Memo, root: MemoGroupId) -> usize { - memo.add_query(MemoQuery { - root: QueryRoot::Operator(root), - accuracy_target: None, - number_of_values_per_query_evaluation: None, - }) -} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests.rs b/crates/stage1-pass1-per-computation-candidates/src/tests.rs index 272151b8a..544ef4948 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/tests.rs @@ -1,36 +1,27 @@ //! Pass 1's tests. Each builds its input memo through `Memo::find_or_add_group` //! and `Memo::add_query`, as Stage 0 does, and reads the candidates //! [`pass1_memo`](crate::pass1_memo) adds; no other stage crate is called. +mod candidates; +mod fused; +mod having; +mod intents; +mod nested_merges; +mod running_example; +mod series_identity; +mod summary_realization; +mod targets; +mod unsupported_ways; +mod update_weights; + use std::fmt::Write as _; -use std::time::Duration; -use asap_types::ir::operator::arguments::{BinaryOpKind, Reduction, Source}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, - TimeRangeKind, -}; -use asap_types::stages::{CandidateForMemoGroup, LogicalRequirements, Memo, MemoGroupId}; -use asap_types::summary_type::{parameter_list, SummaryTypeCapability}; -use asap_types::summary_type::{ - ExactKind, SketchAlgorithm, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, - WeightDomain, -}; - -use crate::candidates::{add_measure_way, key_only_having, Pending, SingleMeasure}; -use crate::computation_targets::{ - computation_candidates_for_intent, computation_targets, HowToComputeAggregateMeasure, -}; -use crate::summary_realization::{ - realize_keyed_additive_summary_input, summarised_input, summary_candidates, -}; -use crate::test_fixtures::{aggregate_group, group, query, scan_group}; -use crate::{pass1_memo, PerComputationCandidateError}; -use asap_types::test_support::memo_fixtures::scan; -use asap_types::workload::DataWorkload; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::{SketchAlgorithm, SummaryUpdate}; +use asap_types::test_support::memo_builders::{group, groups_where, scan_group}; +use std::time::Duration; /// A compact printout of every group and its candidates: the expression's /// operator, then per candidate its operator, input groups, θ and fused @@ -89,20 +80,6 @@ fn label(operator: &Operator) -> String { } } -fn candidates(memo: &Memo, group: MemoGroupId) -> &[CandidateForMemoGroup] { - &memo.groups()[group.0 as usize].candidates -} - -/// The groups of `memo` whose expression matches `pattern`. -fn groups_where(memo: &Memo, pattern: impl Fn(&Operator) -> bool) -> Vec { - memo.groups() - .iter() - .enumerate() - .filter(|(_, group)| pattern(&group.expression)) - .map(|(index, _)| MemoGroupId(index as u32)) - .collect() -} - /// The state group of `algorithm` in `memo`, and its build's update. fn state_group(memo: &Memo, algorithm: SketchAlgorithm) -> (MemoGroupId, SummaryUpdate) { let found = groups_where( @@ -138,7 +115,7 @@ fn topk(k: usize) -> AggIntent { /// `metric[5m]` (`Range`) or `metric` (`Instant`, which a cross-series /// aggregate reads): a selector over `metric{labels}`. fn selector(memo: &mut Memo, kind: TimeRangeKind, metric: &str, labels: &[&str]) -> MemoGroupId { - let scan = scan_group(memo, metric, labels); + let scan = scan_group(memo, metric, labels, vec![]); group( memo, Operator::NonASAP(NonASAPOp::TimeRange { @@ -148,1244 +125,3 @@ fn selector(memo: &mut Memo, kind: TimeRangeKind, metric: &str, labels: &[&str]) }), ) } - -/// `column op literal` over an aggregate's output. -fn compare(column: usize, op: CompareOpKind, value: ScalarValue) -> Predicate { - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op, - right: Box::new(ScalarExpr::Literal(value)), - semantics: ExprSemantics::Promql, - }) -} - -/// `Filter(pred)` over `child`, added as Stage 0 adds a consumer `Filter`. -fn filter(memo: &mut Memo, child: MemoGroupId, pred: Predicate) -> MemoGroupId { - group(memo, Operator::NonASAP(NonASAPOp::Filter { pred, child })) -} - -// ── Running example (concepts doc, section 1) ────────────────────────────── - -/// The doc's running example: query A reads p95 of `latency`, query B -/// reads p99 / p50 of it. KLL state group holds one build per θ of KLL's -/// list, DDSketch's state group likewise; the p95, p99 and p50 groups each -/// read either state through their own readout groups. The metric rule -/// (B's relative ratio cannot read KLL's rank error) drops edges in Pass 2, -/// so Pass 1 offers both algorithms to every quantile group. -#[test] -fn running_example_gives_one_build_per_theta_and_separate_readouts() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = selector(&mut memo, TimeRangeKind::Instant, "latency", &[]); - let g3 = aggregate_group( - &mut memo, - g1, - Reduction::by(vec![]), - vec![quantile(0.95)], - None, - ); - let g4 = aggregate_group( - &mut memo, - g1, - Reduction::by(vec![]), - vec![quantile(0.99)], - None, - ); - let g5 = aggregate_group( - &mut memo, - g1, - Reduction::by(vec![]), - vec![quantile(0.5)], - None, - ); - let g6 = group( - &mut memo, - Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - vector_match: None, - }, - return_bool: false, - lhs: g4, - rhs: g5, - }), - ); - query(&mut memo, g3); - query(&mut memo, g6); - let memo = pass1_memo(memo, None).unwrap(); - println!("{}", memo_printout(&memo)); - let (kll, _) = state_group(&memo, SketchAlgorithm::Kll); - let (ddsketch, _) = state_group(&memo, SketchAlgorithm::DDSketch); - for (state, algorithm) in [ - (kll, SketchAlgorithm::Kll), - (ddsketch, SketchAlgorithm::DDSketch), - ] { - let thetas: Vec = candidates(&memo, state) - .iter() - .map(|candidate| candidate.logical_properties.parameters().unwrap().clone()) - .collect(); - assert_eq!(thetas, parameter_list(algorithm)); - // θ is a property, never an operator argument: every build has the - // group's own expression. - assert!(candidates(&memo, state).iter().all(|candidate| { - candidate.expression.operator == memo.groups()[state.0 as usize].expression - })); - } - // Each quantile group: its exact aggregate and one readout per algorithm, - // each through a readout group of its own over the state group. - for (result, q) in [(g3, 0.95), (g4, 0.99), (g5, 0.5)] { - let readouts: Vec = candidates(&memo, result)[1..] - .iter() - .map(|candidate| *candidate.expression.operator.children()[0]) - .collect(); - assert_eq!(readouts.len(), 2, "G{}", result.0); - for (readout, state) in readouts.iter().zip([kll, ddsketch]) { - assert_eq!( - memo.groups()[readout.0 as usize].expression, - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: state, - query: SketchStatistic::Quantile { q }, - }) - ); - assert!(candidates(&memo, *readout)[0] - .logical_properties - .parameters() - .is_none()); - } - } - // The ratio's group keeps its one candidate, its own expression. - assert_eq!(candidates(&memo, g6).len(), 1); -} - -// ── Series identity ──────────────────────────────────────────────────────── - -/// `quantile_over_time(0.9, latency[5m])` over series whose identity is -/// realized as a column (as the planner does before Stage 0): each readout -/// candidate outputs the per-series aggregate's exact schema, the -/// `$promql_series_identity` column included, so every readout keeps the -/// series it answers for. -#[test] -fn per_series_readouts_keep_the_series_identity() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "latency", &["job"]); - let per_series = aggregate_group( - &mut memo, - rows, - Reduction::PerEntity, - vec![quantile(0.9)], - None, - ); - query(&mut memo, per_series); - let memo = pass1_memo(memo, None).unwrap(); - let output = &memo.groups()[per_series.0 as usize].group_properties.schema; - assert!(output.closed && output.has_promql_series_identity()); - let readouts = &candidates(&memo, per_series)[1..]; - assert_eq!(readouts.len(), 2, "{}", memo_printout(&memo)); - for readout in readouts { - let Operator::NonASAP(NonASAPOp::Project { child, cols, .. }) = - &readout.expression.operator - else { - panic!("a renaming Project: {:?}", readout.expression.operator) - }; - let estimate = &memo.groups()[child.0 as usize].group_properties.schema; - assert!(estimate.has_promql_series_identity()); - let identity = output - .fields - .iter() - .position(|field| field.name == PROMQL_SERIES_IDENTITY) - .unwrap(); - let read = estimate - .fields - .iter() - .position(|field| field.name == PROMQL_SERIES_IDENTITY) - .unwrap(); - assert_eq!(cols[identity].expr, ScalarExpr::Column(read)); - } -} - -/// Without a realized identity the per-series aggregate's schema is open: -/// its rows carry labels it does not list. A renaming `Project` would drop -/// them, so no readout is offered; the group keeps its own expression. -#[test] -fn an_open_per_series_aggregate_offers_no_renamed_readout() { - let mut memo = Memo::new(&DataWorkload::default()); - let scan = group( - &mut memo, - scan("latency", &[("job", DataType::Utf8)], vec![]), - ); - let rows = group( - &mut memo, - Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - }), - ); - let per_series = aggregate_group( - &mut memo, - rows, - Reduction::PerEntity, - vec![quantile(0.9)], - None, - ); - query(&mut memo, per_series); - let memo = pass1_memo(memo, None).unwrap(); - assert!( - !memo.groups()[per_series.0 as usize] - .group_properties - .schema - .closed - ); - assert_eq!(candidates(&memo, per_series).len(), 1); -} - -// ── Target discovery ─────────────────────────────────────────────────────── - -/// Each measure of a multi-measure aggregate is its own target; a measure -/// whose only realization is its exact accumulator is still a target. -#[test] -fn multi_measure_aggregate_has_one_target_per_measure() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let both = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![sum(), quantile(0.9)], - None, - ); - query(&mut memo, both); - let targets = computation_targets(&memo, None).unwrap(); - let measures: Vec<(MemoGroupId, usize)> = targets - .iter() - .map(|target| (target.aggregate_group, target.measure_index)) - .collect(); - assert_eq!(measures, vec![(both, 0), (both, 1)]); -} - -/// An aggregate with no realization besides exact execution (no -/// accumulator, no summary: `stddev`) is not a target. -#[test] -fn aggregate_without_realization_is_not_a_target() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); - let stddev = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![]), - vec![AggIntent::StdDev { - col: None, - population: true, - }], - None, - ); - query(&mut memo, stddev); - assert!(computation_targets(&memo, None).unwrap().is_empty()); - let memo = pass1_memo(memo, None).unwrap(); - assert_eq!(candidates(&memo, stddev).len(), 1); -} - -/// An aggregate several queries read, or a scalar root reads twice, is one -/// target per measure: Pass 1 walks the groups. -#[test] -fn an_aggregate_read_by_several_roots_is_one_target() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); - let p = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![]), - vec![quantile(0.9)], - None, - ); - query(&mut memo, p); - query(&mut memo, p); - assert_eq!(computation_targets(&memo, None).unwrap().len(), 1); -} - -/// A memo whose groups already hold ASAP operators fails at the stage -/// boundary. -#[test] -fn asap_input_is_rejected() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); - let state = group( - &mut memo, - Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - summary_type: FieldDataType::exact(ExactKind::Sum), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }), - ); - query(&mut memo, state); - assert!(matches!( - pass1_memo(memo, None), - Err(PerComputationCandidateError::ContainsAsap) - )); -} - -/// The summaries the deployment does not support are not offered. -#[test] -fn unsupported_summaries_are_not_offered() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &[]); - let p = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![]), - vec![quantile(0.9)], - None, - ); - query(&mut memo, p); - let memo = pass1_memo(memo, Some(&[])).unwrap(); - assert_eq!(candidates(&memo, p).len(), 1); - assert!(groups_where(&memo, |e| matches!(e, Operator::ASAP(_))).is_empty()); -} - -// ── Candidate composition ────────────────────────────────────────────────── - -/// Every group has its own expression as a candidate, with no θ: plain -/// exact execution. -#[test] -fn every_group_offers_its_own_expression() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); - query(&mut memo, s); - let stage0 = memo.groups().len(); - let memo = pass1_memo(memo, None).unwrap(); - for index in 0..stage0 { - let group = &memo.groups()[index]; - let own = &group.candidates[0]; - assert_eq!(own.expression.operator, group.expression); - assert!(own.logical_properties.parameters().is_none()); - } -} - -/// An exact accumulator is a state group (`SummaryAgg` of the exact -/// summary type, no θ) read by a `FinalizeExactAccumulator` group, which the -/// aggregate's group reads through a renaming `Project`. -#[test] -fn exact_accumulator_is_a_state_group_and_a_finalize() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); - query(&mut memo, s); - let memo = pass1_memo(memo, None).unwrap(); - let states = groups_where(&memo, |e| { - matches!( - e, - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }); - let [state] = states[..] else { - panic!("{}", memo_printout(&memo)) - }; - assert_eq!(candidates(&memo, state).len(), 1); - assert!(candidates(&memo, state)[0] - .logical_properties - .parameters() - .is_none()); - // The aggregate reads it through a `FinalizeExactAccumulator`, renamed - // to the aggregate's columns when its names differ. - let finalize = Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }); - let finalize_groups = groups_where(&memo, |e| *e == finalize); - assert!(candidates(&memo, s).iter().any(|c| { - c.expression.operator == finalize - || c.expression - .operator - .children() - .iter() - .any(|g| finalize_groups.contains(g)) - })); -} - -/// A multi-measure aggregate's candidate joins the measures' own groups -/// (one single-measure aggregate each, with its own candidates) on the -/// group keys. -#[test] -fn multi_measure_candidate_joins_single_measure_groups() { - let mut memo = Memo::new(&DataWorkload::default()); - // A SQL table `flows(src Utf8, bytes Float64)`. - let rows = group( - &mut memo, - Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "flows".into(), - }, - predicates: vec![], - schema: Schema::new(vec![ - Field::plain("src", DataType::Utf8, false), - Field::plain("bytes", DataType::Float64, false), - ]), - }), - ); - let both = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![0]), - vec![ - AggIntent::Sum { col: Some(1) }, - AggIntent::Quantile { - col: Some(1), - q: 0.9, - }, - ], - None, - ); - query(&mut memo, both); - let memo = pass1_memo(memo, None).unwrap(); - let single = groups_where( - &memo, - |e| matches!(e, Operator::NonASAP(NonASAPOp::Aggregate { measures, .. }) if measures.len() == 1), - ); - assert_eq!(single.len(), 2, "{}", memo_printout(&memo)); - // The quantile measure's group has its exact aggregate and two readouts. - assert!(single.iter().any(|g| candidates(&memo, *g).len() == 3)); - let joins = groups_where(&memo, |e| { - matches!(e, Operator::NonASAP(NonASAPOp::Join { .. })) - }); - assert_eq!(joins.len(), 1); - assert_eq!(candidates(&memo, both).len(), 2); -} - -/// Pass 1 sets no input requirement, a top-k readout's included: each -/// candidate has one empty requirement per input group, and Pass 2 sets -/// the readouts' required accuracy (a top-k's `heap_size` ≥ k among it). -#[test] -fn pass1_sets_no_input_requirement() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); - let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); - let by_job = aggregate_group( - &mut memo, - per_series, - Reduction::by(vec![2]), - vec![sum()], - None, - ); - let ranked = aggregate_group( - &mut memo, - by_job, - Reduction::by(vec![]), - vec![topk(15)], - None, - ); - let samples = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let p = aggregate_group( - &mut memo, - samples, - Reduction::by(vec![]), - vec![quantile(0.9)], - None, - ); - query(&mut memo, ranked); - query(&mut memo, p); - let memo = pass1_memo(memo, None).unwrap(); - // The top-k readout is offered, and has no input requirement either. - assert!(!groups_where(&memo, |e| matches!( - e, - Operator::ASAP(ASAPOp::SummaryEstimate { - query: SketchStatistic::TopK { k: 15 }, - .. - }) - )) - .is_empty()); - for group in memo.groups() { - for candidate in &group.candidates { - assert_eq!( - candidate.input_requirements, - vec![ - LogicalRequirements::default(); - candidate.expression.operator.children().len() - ] - ); - } - } -} - -// ── HAVING placement ─────────────────────────────────────────────────────── - -/// The aggregate's input with a key-only predicate pushed below it: a -/// `Filter` group over `rows`. -fn pushed_below(memo: &Memo, rows: MemoGroupId) -> Vec { - groups_where( - memo, - |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == rows), - ) -} - -/// A key-only `HAVING` (a consumer `Filter` over the group keys) gets -/// candidates in the `Filter`'s group that push it below the aggregate: a -/// `Filter` on the input rows, which the states read. The aggregate's own -/// group keeps its unfiltered candidates for its other consumers. -#[test] -fn key_only_having_is_pushed_below_into_the_filter_group() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let by_job = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![quantile(0.9)], - None, - ); - let having = filter( - &mut memo, - by_job, - compare(0, CompareOpKind::Eq, ScalarValue::Utf8("a".into())), - ); - query(&mut memo, having); - query(&mut memo, by_job); - let memo = pass1_memo(memo, None).unwrap(); - let below = pushed_below(&memo, rows); - assert_eq!(below.len(), 1, "{}", memo_printout(&memo)); - // The Filter group: its own expression, then one readout per algorithm - // over states built from the filtered rows. - assert_eq!(candidates(&memo, having).len(), 3); - assert_eq!(candidates(&memo, by_job).len(), 3); -} - -/// A `HAVING` over an aggregate value stays above the output: nothing is -/// pushed below, and the `Filter` group keeps its own expression only. -#[test] -fn value_having_stays_above() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let by_job = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![quantile(0.9)], - None, - ); - let having = filter( - &mut memo, - by_job, - compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), - ); - query(&mut memo, having); - let memo = pass1_memo(memo, None).unwrap(); - assert!(pushed_below(&memo, rows).is_empty()); - assert_eq!(candidates(&memo, having).len(), 1); -} - -/// A global aggregate's `HAVING` stays above even when it is constant: the -/// one output row exists with no input row. -#[test] -fn global_aggregate_having_stays_above() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); - let p = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![]), - vec![quantile(0.9)], - None, - ); - let having = filter( - &mut memo, - p, - Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))), - ); - query(&mut memo, having); - let memo = pass1_memo(memo, None).unwrap(); - assert!(pushed_below(&memo, rows).is_empty()); -} - -/// Which `HAVING` conjuncts of a grouped aggregate (`by (job)`) are pushed -/// below it: a key conjunct, and a column-free one built of literals and -/// comparison and boolean operators (`1 = 0`); not a column-free one with a -/// function call, which may be volatile (`random() < 0.5`, built in the IR -/// directly: the frontends and schema derivation reject `random`), nor -/// one over the aggregate's value, nor any of a global aggregate's. -#[test] -fn key_only_having_pushes_keys_and_constants_but_no_function_call() { - let literal = |value| Box::new(ScalarExpr::Literal(value)); - let one_equals_zero = ScalarExpr::Compare { - left: literal(ScalarValue::Int64(1)), - op: CompareOpKind::Eq, - right: literal(ScalarValue::Int64(0)), - semantics: ExprSemantics::Sql, - }; - let random_below_half = ScalarExpr::Compare { - left: Box::new(ScalarExpr::FunctionCall { - name: "random".into(), - args: vec![], - }), - op: CompareOpKind::Lt, - right: literal(ScalarValue::Float64(0.5)), - semantics: ExprSemantics::Sql, - }; - let job_is_a = compare(0, CompareOpKind::Eq, ScalarValue::Utf8("a".into())).0; - let value_above_ten = compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)).0; - let by_job = Reduction::by(vec![2]); - assert!(key_only_having(&by_job, &job_is_a)); - assert!(key_only_having(&by_job, &one_equals_zero)); - assert!(key_only_having( - &by_job, - &ScalarExpr::Not(Box::new(one_equals_zero.clone())) - )); - assert!(!key_only_having(&by_job, &random_below_half)); - assert!(!key_only_having(&by_job, &value_above_ten)); - let global = Reduction::by(vec![]); - assert!(!key_only_having(&global, &one_equals_zero)); -} - -/// A grouped aggregate's constant `HAVING 1 = 0` is pushed below it. -#[test] -fn a_constant_having_of_a_grouped_aggregate_is_pushed_below() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let by_job = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![quantile(0.9)], - None, - ); - let having = filter( - &mut memo, - by_job, - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Literal(ScalarValue::Int64(1))), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), - semantics: ExprSemantics::Sql, - }), - ); - query(&mut memo, having); - let memo = pass1_memo(memo, None).unwrap(); - assert_eq!( - pushed_below(&memo, rows).len(), - 1, - "{}", - memo_printout(&memo) - ); - assert_eq!(candidates(&memo, having).len(), 3); -} - -/// The aggregate's own value `having` is a `Filter` above each readout: -/// the aggregate group's summary candidates are `Filter`s over a readout. -#[test] -fn own_value_having_filters_each_readout() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let by_job = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![quantile(0.9)], - Some(compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0))), - ); - query(&mut memo, by_job); - let memo = pass1_memo(memo, None).unwrap(); - let kinds: Vec<&str> = candidates(&memo, by_job) - .iter() - .map(|c| c.expression.operator.kind_name()) - .collect(); - assert_eq!(kinds, ["Aggregate", "Filter", "Filter"]); -} - -// ── Fused heaps, whole-expression top-k, ranked items, raw-data signs ────── - -/// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the top-k gets heaps -/// over the range's raw data samples, keyed by `job` (the ranked item), -/// weighted by the sample value with the open requirement that the samples -/// of `m` are non-negative (`RawDataSamplesOfMetric`). Each heap build -/// lists the absorbed sums' groups in `fused_groups`, and the top-k's own -/// heaps over its one-row-per-item input are pruned. -#[test] -fn whole_expression_topk_heaps_read_the_raw_samples() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); - let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); - let by_job = aggregate_group( - &mut memo, - per_series, - Reduction::by(vec![2]), - vec![sum()], - None, - ); - let ranked = aggregate_group( - &mut memo, - by_job, - Reduction::by(vec![]), - vec![topk(5)], - None, - ); - query(&mut memo, ranked); - let memo = pass1_memo(memo, None).unwrap(); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - let (state, update) = state_group(&memo, algorithm); - assert_eq!( - *memo.groups()[state.0 as usize].expression.children()[0], - rows - ); - assert_eq!( - update.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) - ); - assert_eq!( - update.weight_domain, - WeightDomain::RawDataSamplesOfMetric { metric: "m".into() } - ); - for build in candidates(&memo, state) { - let mut fused = build.fused_groups.clone(); - fused.sort(); - assert_eq!(fused, vec![per_series, by_job]); - } - assert_eq!( - candidates(&memo, state).len(), - parameter_list(algorithm).len() - ); - } -} - -/// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the whole-expression -/// heap's candidates, built through the readout and output path every -/// summary way takes. The top-k group gets one `Project` per heap -/// algorithm, ranking the estimate read from that heap's state; only the -/// state's builds list the absorbed groups. -#[test] -fn whole_expression_topk_heap_candidates() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); - let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); - let by_job = aggregate_group( - &mut memo, - per_series, - Reduction::by(vec![2]), - vec![sum()], - None, - ); - let ranked = aggregate_group( - &mut memo, - by_job, - Reduction::by(vec![]), - vec![topk(5)], - None, - ); - query(&mut memo, ranked); - let memo = pass1_memo(memo, None).unwrap(); - // The top-k group's own expression, then one ranked `Project` per heap. - let ranked_candidates = candidates(&memo, ranked); - let kinds: Vec<&str> = ranked_candidates - .iter() - .map(|c| c.expression.operator.kind_name()) - .collect(); - assert_eq!( - kinds, - ["Aggregate", "Project", "Project"], - "{}", - memo_printout(&memo) - ); - for (project, algorithm) in ranked_candidates[1..].iter().zip([ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ]) { - assert!(project.fused_groups.is_empty()); - let estimate = *project.expression.operator.children()[0]; - let [estimate_candidate] = candidates(&memo, estimate) else { - panic!("one readout candidate: {}", memo_printout(&memo)); - }; - assert!(matches!( - estimate_candidate.expression.operator, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - )); - assert!(estimate_candidate.fused_groups.is_empty()); - let (state, _) = state_group(&memo, algorithm); - assert_eq!(*estimate_candidate.expression.operator.children()[0], state); - assert_eq!( - candidates(&memo, state).len(), - parameter_list(algorithm).len() - ); - for build in candidates(&memo, state) { - assert_eq!(build.fused_groups, [by_job, per_series]); - } - } - // Scan, range, the two sums, the top-k, the sums' two accumulator - // states, and per heap its state and readout: no other group. - assert_eq!(memo.groups().len(), 11, "{}", memo_printout(&memo)); -} - -/// A top-k over a `count` ranks the counted items with unit weight, proven -/// non-negative. -#[test] -fn count_ranked_topk_counts_with_unit_weight() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let counts = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![AggIntent::Count], - None, - ); - let ranked = aggregate_group( - &mut memo, - counts, - Reduction::by(vec![]), - vec![topk(5)], - None, - ); - query(&mut memo, ranked); - let memo = pass1_memo(memo, None).unwrap(); - let (_, update) = state_group(&memo, SketchAlgorithm::CmsWithHeap); - assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - assert!(matches!( - update.weight_domain, - WeightDomain::NonNegative { .. } - )); -} - -/// The ranked item over several group keys is their tuple; the top-k's own -/// partition keys partition the heap and stay out of the tuple. -#[test] -fn ranked_item_is_the_keys_outside_the_partition() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job", "host"]); - let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); - let by_both = aggregate_group( - &mut memo, - per_series, - Reduction::by(vec![2, 3]), - vec![sum()], - None, - ); - let ranked = aggregate_group( - &mut memo, - by_both, - Reduction::by(vec![0]), - vec![topk(5)], - None, - ); - query(&mut memo, ranked); - let memo = pass1_memo(memo, None).unwrap(); - let (_, update) = state_group(&memo, SketchAlgorithm::CountSketchWithHeap); - assert_eq!( - update.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))) - ); -} - -// ── Nested distributive merges ───────────────────────────────────────────── - -/// `sum by (job) (sum_over_time(m[5m]))`, and likewise `min` of `min` and -/// `max` of `max`, gets a merged build: one exact accumulator of that kind -/// over the raw samples grouped by `job`, a candidate of the outer -/// aggregate's own accumulator state group, listing the inner aggregate's -/// group in `fused_groups`. -#[test] -fn nested_distributive_aggregate_merges_into_the_outer_state_group() { - for (intent, kind) in [ - (sum(), ExactKind::Sum), - (AggIntent::Min { col: None }, ExactKind::Min), - (AggIntent::Max { col: None }, ExactKind::Max), - ] { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); - let per_series = aggregate_group( - &mut memo, - rows, - Reduction::PerEntity, - vec![intent.clone()], - None, - ); - let by_job = aggregate_group( - &mut memo, - per_series, - Reduction::by(vec![2]), - vec![intent], - None, - ); - query(&mut memo, by_job); - let memo = pass1_memo(memo, None).unwrap(); - let outer_state = groups_where( - &memo, - |e| matches!(e, Operator::ASAP(ASAPOp::SummaryAgg { child, summary_type: FieldDataType::ExactAggregate(state, _), .. }) if *child == per_series && *state == kind), - ); - let [outer_state] = outer_state[..] else { - panic!("{kind:?}: {}", memo_printout(&memo)) - }; - let builds = candidates(&memo, outer_state); - assert_eq!(builds.len(), 2, "{kind:?}: {}", memo_printout(&memo)); - let merged = &builds[1]; - assert_eq!(merged.expression.operator.children(), vec![&rows]); - assert_eq!(merged.fused_groups, vec![per_series]); - } -} - -/// The merge needs the same distributive aggregate twice: a `sum` over a -/// `max`, or a `min` over a `max`, gets none. -#[test] -fn nested_merge_needs_the_same_distributive_aggregate() { - for outer in [sum(), AggIntent::Min { col: None }] { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); - let per_series = aggregate_group( - &mut memo, - rows, - Reduction::PerEntity, - vec![AggIntent::Max { col: None }], - None, - ); - let by_job = aggregate_group( - &mut memo, - per_series, - Reduction::by(vec![2]), - vec![outer], - None, - ); - query(&mut memo, by_job); - let memo = pass1_memo(memo, None).unwrap(); - assert!(memo - .groups() - .iter() - .flat_map(|g| &g.candidates) - .all(|c| c.fused_groups.is_empty())); - } -} - -// ── Stream norms (update weights) ────────────────────────────────────────── - -/// A Count-Min count adds one per row (bounded by the row count); a sum's -/// state reads the raw value column (bounded by the source's norm). -#[test] -fn update_weights_follow_the_measure() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let counts = aggregate_group( - &mut memo, - rows, - Reduction::by(vec![2]), - vec![AggIntent::Count], - None, - ); - query(&mut memo, counts); - let memo = pass1_memo(memo, None).unwrap(); - let (_, update) = state_group(&memo, SketchAlgorithm::Cms); - assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - let mut memo = Memo::new(&DataWorkload::default()); - let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); - let sums = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); - query(&mut memo, sums); - let memo = pass1_memo(memo, None).unwrap(); - let state = groups_where( - &memo, - |e| matches!(e, Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) if input.weight == SummaryInputExpr::Column(ColumnRef::SampleValue)), - ); - assert_eq!(state.len(), 1); -} - -// ── Intents ──────────────────────────────────────────────────────────────── - -fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec { - choices - .iter() - .filter_map(|choice| match choice { - HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), - _ => None, - }) - .collect() -} - -/// Example 2 preserves specialized distinct summaries and the universal -/// alternative; plain exact execution is the group's own expression. -#[test] -fn cardinality_keeps_specialized_and_universal_summaries() { - let choices = computation_candidates_for_intent(&AggIntent::Cardinality { cols: vec![0] }); - assert!(choices - .iter() - .all(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_)))); - assert_eq!( - algorithms(&choices), - vec![ - SketchAlgorithm::Hll, - SketchAlgorithm::Kmv, - SketchAlgorithm::UnivMon - ] - ); - let tuple = computation_candidates_for_intent(&AggIntent::Cardinality { cols: vec![0, 1] }); - assert!(!algorithms(&tuple).contains(&SketchAlgorithm::UnivMon)); -} - -/// Frequency moments offer a universal sketch without certification. -#[test] -fn frequency_statistics_keep_universal_choices() { - for intent in [ - AggIntent::FrequencyL2 { col: Some(0) }, - AggIntent::FrequencyEntropy { col: Some(0) }, - ] { - let choices = computation_candidates_for_intent(&intent); - assert_eq!( - choices, - [HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::UnivMon - )] - ); - } -} - -/// TopK keeps both declared heap substrates without choosing an implementation. -#[test] -fn topk_keeps_both_specialized_heap_choices() { - let choices = computation_candidates_for_intent(&topk(10)); - assert_eq!( - algorithms(&choices), - vec![ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap - ] - ); -} - -/// Approximate count lists its exact accumulator first, beside the -/// summaries (the plain aggregate is the group's own expression). -#[test] -fn approximate_count_lists_its_accumulator_and_summaries() { - let choices = computation_candidates_for_intent(&AggIntent::Count); - assert_eq!( - choices[0], - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]) - ); - assert!(choices.contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::UnivMon - ))); -} - -/// Count Sketch's signed counters give no total, so Pass 1 offers no Count -/// Sketch for a count. -#[test] -fn a_count_offers_no_count_sketch() { - assert!( - !computation_candidates_for_intent(&AggIntent::Count).contains( - &HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketch) - ) - ); -} - -// ── Summary realization ────────────────────────────────────────────────────── - -/// A quantile outside [0, 1] is answered -Inf or +Inf: it gets no summary, -/// so no way besides the plain exact aggregate. -#[test] -fn a_quantile_outside_the_unit_interval_offers_no_summary() { - let choices = computation_candidates_for_intent(&AggIntent::Quantile { col: None, q: 1.5 }); - assert!(choices.is_empty()); -} - -/// A heap sketch consumes every raw row, so the keyed input rule refuses -/// a TopK over a filtered count. -#[test] -fn keyed_heap_input_refuses_a_filtered_count() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = scan_group(&mut memo, "m", &["job"]); - let mut count_by_job = |filter: Option>| { - group( - &mut memo, - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Count], - output_names: vec![String::new()], - filters: vec![filter], - having: None, - child: rows, - }), - ) - }; - let unfiltered = count_by_job(None); - let value_above_one = Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(1)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: ExprSemantics::Promql, - }); - let filtered = count_by_job(Some(value_above_one)); - let output = Reduction::Reduce(asap_types::ir::operator::arguments::GroupKeys::none()); - assert!(realize_keyed_additive_summary_input(&memo, &output, unfiltered).is_some()); - assert!(realize_keyed_additive_summary_input(&memo, &output, filtered).is_none()); -} - -/// A summary reads the intent's own input columns: one column by name, -/// none as the sample value, several as one hashed tuple. -#[test] -fn summarised_input_reads_the_intents_columns() { - // Columns [ts=0, value=1, job=2]. - let Operator::NonASAP(NonASAPOp::Scan { schema, .. }) = - scan("m", &[("job", DataType::Utf8)], vec![]) - else { - unreachable!("a scan") - }; - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - let cases = [ - (vec![2], column("job")), - (vec![1], column("value")), - (vec![], SummaryInputExpr::Column(ColumnRef::SampleValue)), - ( - vec![1, 2], - SummaryInputExpr::Tuple(vec![column("value"), column("job")]), - ), - ]; - for (cols, want) in cases { - let intent = AggIntent::Cardinality { cols: cols.clone() }; - assert_eq!( - summarised_input(&intent, &schema), - Some(want), - "wrong summarised input for cols={cols:?}" - ); - } -} - -/// Each approximable intent is offered the algorithms it was offered when -/// the list was written by hand, in the same order; a non-sketch intent -/// is offered none. -#[test] -fn summary_candidates_keep_the_hand_written_lists() { - use SketchAlgorithm as S; - let cardinality = |cols: Vec| AggIntent::Cardinality { cols }; - let cases = [ - ( - AggIntent::Quantile { col: None, q: 0.99 }, - vec![S::Kll, S::DDSketch], - ), - (cardinality(vec![]), vec![S::Hll, S::Kmv, S::UnivMon]), - (cardinality(vec![1]), vec![S::Hll, S::Kmv, S::UnivMon]), - (cardinality(vec![1, 2]), vec![S::Hll, S::Kmv]), - (AggIntent::FrequencyL2 { col: None }, vec![S::UnivMon]), - (AggIntent::FrequencyEntropy { col: None }, vec![S::UnivMon]), - ( - AggIntent::TopK { k: 10 }, - vec![S::CmsWithHeap, S::CountSketchWithHeap], - ), - (AggIntent::Count, vec![S::Cms, S::UnivMon]), - (AggIntent::Sum { col: None }, vec![]), - ]; - for (intent, want) in cases { - assert_eq!(summary_candidates(&intent), want, "{intent:?}"); - } -} - -/// An algorithm is offered for an intent exactly when its capability table -/// row answers the intent's statistic: Count-Min's row has no top-k readout, -/// so a top-k is offered only the heap variants, whose rows have one. -#[test] -fn an_algorithm_without_the_readout_is_not_offered() { - let top_k = AggIntent::TopK { k: 10 }; - let statistic = SketchStatistic::TopK { k: 10 }; - let answers = |algorithm| { - SummaryTypeCapability::of(&FieldDataType::Sketch(algorithm)).answers(&statistic) - }; - assert!(!answers(SketchAlgorithm::Cms)); - assert!(answers(SketchAlgorithm::CmsWithHeap)); - let offered = summary_candidates(&top_k); - assert!(!offered.contains(&SketchAlgorithm::Cms)); - assert!(offered.iter().all(|algorithm| answers(*algorithm))); -} - -// ── Unsupported ways add nothing ───────────────────────────────────────────── - -/// Offers `measure` by `way` to `result_group`, expecting `Unsupported`, and -/// checks the memo is unchanged: its groups, each group's candidates and -/// its hash-consing index (all in the memo's `Debug`). -fn assert_unsupported_way_adds_nothing( - memo: &mut Memo, - measure: &SingleMeasure<'_>, - way: HowToComputeAggregateMeasure, - result_group: MemoGroupId, -) { - let groups = memo.groups().len(); - let before = format!("{memo:?}"); - let added = add_measure_way(memo, Pending::new(memo), measure, &way, result_group, None); - assert!( - matches!(added, Err(PerComputationCandidateError::Unsupported(_))), - "{added:?}" - ); - assert_eq!(memo.groups().len(), groups); - assert_eq!(format!("{memo:?}"), before); -} - -/// A heap top-k whose estimate lacks a column of the result group (it -/// partitions `host` rows; the group ranks `job`): `ranked_as_topk` finds no -/// `job` in the pending estimate, so neither the heap's state group nor its -/// estimate group is added. -#[test] -fn a_heap_estimate_without_the_groups_columns_adds_nothing() { - let mut memo = Memo::new(&DataWorkload::default()); - let by_job = scan_group(&mut memo, "m", &["job"]); - let by_host = scan_group(&mut memo, "n", &["host"]); - let ranked = aggregate_group( - &mut memo, - by_job, - Reduction::by(vec![2]), - vec![topk(5)], - None, - ); - let reduction = Reduction::by(vec![2]); - let intent = topk(5); - let measure = SingleMeasure { - input: by_host, - unfiltered_input: by_host, - reduction: &reduction, - intent: &intent, - filter: None, - raw_data_samples_of: None, - }; - assert_unsupported_way_adds_nothing( - &mut memo, - &measure, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - ranked, - ); -} - -/// A readout of an open per-series aggregate would need a renaming -/// `Project`, which drops the labels an open schema does not list: the -/// KLL state and readout groups are not added. -#[test] -fn a_readout_an_open_group_cannot_rename_adds_nothing() { - let mut memo = Memo::new(&DataWorkload::default()); - let scanned = group( - &mut memo, - scan("latency", &[("job", DataType::Utf8)], vec![]), - ); - let rows = group( - &mut memo, - Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scanned, - }), - ); - let intent = quantile(0.9); - let per_series = aggregate_group( - &mut memo, - rows, - Reduction::PerEntity, - vec![intent.clone()], - None, - ); - let measure = SingleMeasure { - input: rows, - unfiltered_input: rows, - reduction: &Reduction::PerEntity, - intent: &intent, - filter: None, - raw_data_samples_of: None, - }; - assert_unsupported_way_adds_nothing( - &mut memo, - &measure, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - per_series, - ); -} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/candidates.rs new file mode 100644 index 000000000..807085b80 --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/candidates.rs @@ -0,0 +1,174 @@ +use super::{memo_printout, quantile, selector, sum, topk}; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{LogicalRequirements, Memo}; +use asap_types::summary_type::{ExactKind, SketchStatistic}; +use asap_types::test_support::memo_builders::{candidates_of, group, groups_where, query}; +use asap_types::workload::DataWorkload; + +/// Every group has its own expression as a candidate, with no θ: plain +/// exact execution. +#[test] +fn every_group_offers_its_own_expression() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); + query(&mut memo, s, None); + let stage0 = memo.groups().len(); + let memo = pass1_memo(memo, None).unwrap(); + for index in 0..stage0 { + let group = &memo.groups()[index]; + let own = &group.candidates[0]; + assert_eq!(own.expression.operator, group.expression); + assert!(own.logical_properties.parameters().is_none()); + } +} + +/// An exact accumulator is a state group (`SummaryAgg` of the exact +/// summary type, no θ) read by a `FinalizeExactAccumulator` group, which the +/// aggregate's group reads through a renaming `Project`. +#[test] +fn exact_accumulator_is_a_state_group_and_a_finalize() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); + query(&mut memo, s, None); + let memo = pass1_memo(memo, None).unwrap(); + let states = groups_where(&memo, |e| { + matches!( + e, + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), + .. + }) + ) + }); + let [state] = states[..] else { + panic!("{}", memo_printout(&memo)) + }; + assert_eq!(candidates_of(&memo, state).len(), 1); + assert!(candidates_of(&memo, state)[0] + .logical_properties + .parameters() + .is_none()); + // The aggregate reads it through a `FinalizeExactAccumulator`, renamed + // to the aggregate's columns when its names differ. + let finalize = Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }); + let finalize_groups = groups_where(&memo, |e| *e == finalize); + assert!(candidates_of(&memo, s).iter().any(|c| { + c.expression.operator == finalize + || c.expression + .operator + .children() + .iter() + .any(|g| finalize_groups.contains(g)) + })); +} + +/// A multi-measure aggregate's candidate joins the measures' own groups +/// (one single-measure aggregate each, with its own candidates) on the +/// group keys. +#[test] +fn multi_measure_candidate_joins_single_measure_groups() { + let mut memo = Memo::new(&DataWorkload::default()); + // A SQL table `flows(src Utf8, bytes Float64)`. + let rows = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "flows".into(), + }, + predicates: vec![], + schema: Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }), + ); + let both = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![0]), + vec![ + AggIntent::Sum { col: Some(1) }, + AggIntent::Quantile { + col: Some(1), + q: 0.9, + }, + ], + None, + ); + query(&mut memo, both, None); + let memo = pass1_memo(memo, None).unwrap(); + let single = groups_where( + &memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Aggregate { measures, .. }) if measures.len() == 1), + ); + assert_eq!(single.len(), 2, "{}", memo_printout(&memo)); + // The quantile measure's group has its exact aggregate and two readouts. + assert!(single.iter().any(|g| candidates_of(&memo, *g).len() == 3)); + let joins = groups_where(&memo, |e| { + matches!(e, Operator::NonASAP(NonASAPOp::Join { .. })) + }); + assert_eq!(joins.len(), 1); + assert_eq!(candidates_of(&memo, both).len(), 2); +} + +/// Pass 1 sets no input requirement, a top-k readout's included: each +/// candidate has one empty requirement per input group, and Pass 2 sets +/// the readouts' required accuracy (a top-k's `heap_size` ≥ k among it). +#[test] +fn pass1_sets_no_input_requirement() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_job, + Reduction::by(vec![]), + vec![topk(15)], + None, + ); + let samples = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let p = aggregate_group( + &mut memo, + samples, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + query(&mut memo, ranked, None); + query(&mut memo, p, None); + let memo = pass1_memo(memo, None).unwrap(); + // The top-k readout is offered, and has no input requirement either. + assert!(!groups_where(&memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryEstimate { + query: SketchStatistic::TopK { k: 15 }, + .. + }) + )) + .is_empty()); + for group in memo.groups() { + for candidate in &group.candidates { + assert_eq!( + candidate.input_requirements, + vec![ + LogicalRequirements::default(); + candidate.expression.operator.children().len() + ] + ); + } + } +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs new file mode 100644 index 000000000..f08d482ee --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs @@ -0,0 +1,194 @@ +use super::{memo_printout, selector, state_group, sum, topk}; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::{ASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::Memo; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryInputExpr, WeightDomain}; +use asap_types::test_support::memo_builders::{candidates_of, query}; +use asap_types::workload::DataWorkload; + +/// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the top-k gets heaps +/// over the range's raw data samples, keyed by `job` (the ranked item), +/// weighted by the sample value with the open requirement that the samples +/// of `m` are non-negative (`RawDataSamplesOfMetric`). Each heap build +/// lists the absorbed sums' groups in `fused_groups`, and the top-k's own +/// heaps over its one-row-per-item input are pruned. +#[test] +fn whole_expression_topk_heaps_read_the_raw_samples() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_job, + Reduction::by(vec![]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked, None); + let memo = pass1_memo(memo, None).unwrap(); + for algorithm in [ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ] { + let (state, update) = state_group(&memo, algorithm); + assert_eq!( + *memo.groups()[state.0 as usize].expression.children()[0], + rows + ); + assert_eq!( + update.item, + Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) + ); + assert_eq!( + update.weight_domain, + WeightDomain::RawDataSamplesOfMetric { metric: "m".into() } + ); + for build in candidates_of(&memo, state) { + let mut fused = build.fused_groups.clone(); + fused.sort(); + assert_eq!(fused, vec![per_series, by_job]); + } + assert_eq!( + candidates_of(&memo, state).len(), + parameter_list(algorithm).len() + ); + } +} + +/// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the whole-expression +/// heap's candidates, built through the readout and output path every +/// summary way takes. The top-k group gets one `Project` per heap +/// algorithm, ranking the estimate read from that heap's state; only the +/// state's builds list the absorbed groups. +#[test] +fn whole_expression_topk_heap_candidates() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_job, + Reduction::by(vec![]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked, None); + let memo = pass1_memo(memo, None).unwrap(); + // The top-k group's own expression, then one ranked `Project` per heap. + let ranked_candidates = candidates_of(&memo, ranked); + let kinds: Vec<&str> = ranked_candidates + .iter() + .map(|c| c.expression.operator.kind_name()) + .collect(); + assert_eq!( + kinds, + ["Aggregate", "Project", "Project"], + "{}", + memo_printout(&memo) + ); + for (project, algorithm) in ranked_candidates[1..].iter().zip([ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ]) { + assert!(project.fused_groups.is_empty()); + let estimate = *project.expression.operator.children()[0]; + let [estimate_candidate] = candidates_of(&memo, estimate) else { + panic!("one readout candidate: {}", memo_printout(&memo)); + }; + assert!(matches!( + estimate_candidate.expression.operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + )); + assert!(estimate_candidate.fused_groups.is_empty()); + let (state, _) = state_group(&memo, algorithm); + assert_eq!(*estimate_candidate.expression.operator.children()[0], state); + assert_eq!( + candidates_of(&memo, state).len(), + parameter_list(algorithm).len() + ); + for build in candidates_of(&memo, state) { + assert_eq!(build.fused_groups, [by_job, per_series]); + } + } + // Scan, range, the two sums, the top-k, the sums' two accumulator + // states, and per heap its state and readout: no other group. + assert_eq!(memo.groups().len(), 11, "{}", memo_printout(&memo)); +} + +/// A top-k over a `count` ranks the counted items with unit weight, proven +/// non-negative. +#[test] +fn count_ranked_topk_counts_with_unit_weight() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let counts = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![AggIntent::Count], + None, + ); + let ranked = aggregate_group( + &mut memo, + counts, + Reduction::by(vec![]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked, None); + let memo = pass1_memo(memo, None).unwrap(); + let (_, update) = state_group(&memo, SketchAlgorithm::CmsWithHeap); + assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); + assert!(matches!( + update.weight_domain, + WeightDomain::NonNegative { .. } + )); +} + +/// The ranked item over several group keys is their tuple; the top-k's own +/// partition keys partition the heap and stay out of the tuple. +#[test] +fn ranked_item_is_the_keys_outside_the_partition() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job", "host"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_both = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2, 3]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_both, + Reduction::by(vec![0]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked, None); + let memo = pass1_memo(memo, None).unwrap(); + let (_, update) = state_group(&memo, SketchAlgorithm::CountSketchWithHeap); + assert_eq!( + update.item, + Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))) + ); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/having.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/having.rs new file mode 100644 index 000000000..2e1a97802 --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/having.rs @@ -0,0 +1,206 @@ +use super::{memo_printout, quantile, selector}; +use crate::candidates::key_only_having; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; +use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::test_support::memo_builders::{candidates_of, group, groups_where, query}; +use asap_types::workload::DataWorkload; + +/// `column op literal` over an aggregate's output. +fn compare(column: usize, op: CompareOpKind, value: ScalarValue) -> Predicate { + Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Promql, + }) +} + +/// `Filter(pred)` over `child`, added as Stage 0 adds a consumer `Filter`. +fn filter(memo: &mut Memo, child: MemoGroupId, pred: Predicate) -> MemoGroupId { + group(memo, Operator::NonASAP(NonASAPOp::Filter { pred, child })) +} + +/// The aggregate's input with a key-only predicate pushed below it: a +/// `Filter` group over `rows`. +fn pushed_below(memo: &Memo, rows: MemoGroupId) -> Vec { + groups_where( + memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == rows), + ) +} + +/// A key-only `HAVING` (a consumer `Filter` over the group keys) gets +/// candidates in the `Filter`'s group that push it below the aggregate: a +/// `Filter` on the input rows, which the states read. The aggregate's own +/// group keeps its unfiltered candidates for its other consumers. +#[test] +fn key_only_having_is_pushed_below_into_the_filter_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + by_job, + compare(0, CompareOpKind::Eq, ScalarValue::Utf8("a".into())), + ); + query(&mut memo, having, None); + query(&mut memo, by_job, None); + let memo = pass1_memo(memo, None).unwrap(); + let below = pushed_below(&memo, rows); + assert_eq!(below.len(), 1, "{}", memo_printout(&memo)); + // The Filter group: its own expression, then one readout per algorithm + // over states built from the filtered rows. + assert_eq!(candidates_of(&memo, having).len(), 3); + assert_eq!(candidates_of(&memo, by_job).len(), 3); +} + +/// A `HAVING` over an aggregate value stays above the output: nothing is +/// pushed below, and the `Filter` group keeps its own expression only. +#[test] +fn value_having_stays_above() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + by_job, + compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), + ); + query(&mut memo, having, None); + let memo = pass1_memo(memo, None).unwrap(); + assert!(pushed_below(&memo, rows).is_empty()); + assert_eq!(candidates_of(&memo, having).len(), 1); +} + +/// A global aggregate's `HAVING` stays above even when it is constant: the +/// one output row exists with no input row. +#[test] +fn global_aggregate_having_stays_above() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let p = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + p, + Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))), + ); + query(&mut memo, having, None); + let memo = pass1_memo(memo, None).unwrap(); + assert!(pushed_below(&memo, rows).is_empty()); +} + +/// Which `HAVING` conjuncts of a grouped aggregate (`by (job)`) are pushed +/// below it: a key conjunct, and a column-free one built of literals and +/// comparison and boolean operators (`1 = 0`); not a column-free one with a +/// function call, which may be volatile (`random() < 0.5`, built in the IR +/// directly: the frontends and schema derivation reject `random`), nor +/// one over the aggregate's value, nor any of a global aggregate's. +#[test] +fn key_only_having_pushes_keys_and_constants_but_no_function_call() { + let literal = |value| Box::new(ScalarExpr::Literal(value)); + let one_equals_zero = ScalarExpr::Compare { + left: literal(ScalarValue::Int64(1)), + op: CompareOpKind::Eq, + right: literal(ScalarValue::Int64(0)), + semantics: ExprSemantics::Sql, + }; + let random_below_half = ScalarExpr::Compare { + left: Box::new(ScalarExpr::FunctionCall { + name: "random".into(), + args: vec![], + }), + op: CompareOpKind::Lt, + right: literal(ScalarValue::Float64(0.5)), + semantics: ExprSemantics::Sql, + }; + let job_is_a = compare(0, CompareOpKind::Eq, ScalarValue::Utf8("a".into())).0; + let value_above_ten = compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)).0; + let by_job = Reduction::by(vec![2]); + assert!(key_only_having(&by_job, &job_is_a)); + assert!(key_only_having(&by_job, &one_equals_zero)); + assert!(key_only_having( + &by_job, + &ScalarExpr::Not(Box::new(one_equals_zero.clone())) + )); + assert!(!key_only_having(&by_job, &random_below_half)); + assert!(!key_only_having(&by_job, &value_above_ten)); + let global = Reduction::by(vec![]); + assert!(!key_only_having(&global, &one_equals_zero)); +} + +/// A grouped aggregate's constant `HAVING 1 = 0` is pushed below it. +#[test] +fn a_constant_having_of_a_grouped_aggregate_is_pushed_below() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + by_job, + Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Literal(ScalarValue::Int64(1))), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: ExprSemantics::Sql, + }), + ); + query(&mut memo, having, None); + let memo = pass1_memo(memo, None).unwrap(); + assert_eq!( + pushed_below(&memo, rows).len(), + 1, + "{}", + memo_printout(&memo) + ); + assert_eq!(candidates_of(&memo, having).len(), 3); +} + +/// The aggregate's own value `having` is a `Filter` above each readout: +/// the aggregate group's summary candidates are `Filter`s over a readout. +#[test] +fn own_value_having_filters_each_readout() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + Some(compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0))), + ); + query(&mut memo, by_job, None); + let memo = pass1_memo(memo, None).unwrap(); + let kinds: Vec<&str> = candidates_of(&memo, by_job) + .iter() + .map(|c| c.expression.operator.kind_name()) + .collect(); + assert_eq!(kinds, ["Aggregate", "Filter", "Filter"]); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/intents.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/intents.rs new file mode 100644 index 000000000..a2dcd75da --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/intents.rs @@ -0,0 +1,87 @@ +use super::topk; +use crate::computation_targets::{ways_to_compute_intent, HowToComputeAggregateMeasure}; +use asap_types::ir::operator::AggIntent; +use asap_types::summary_type::{ExactKind, SketchAlgorithm}; + +fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec { + choices + .iter() + .filter_map(|choice| match choice { + HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), + _ => None, + }) + .collect() +} + +/// Example 2 preserves specialized distinct summaries and the universal +/// alternative; plain exact execution is the group's own expression. +#[test] +fn cardinality_keeps_specialized_and_universal_summaries() { + let choices = ways_to_compute_intent(&AggIntent::Cardinality { cols: vec![0] }); + assert!(choices + .iter() + .all(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_)))); + assert_eq!( + algorithms(&choices), + vec![ + SketchAlgorithm::Hll, + SketchAlgorithm::Kmv, + SketchAlgorithm::UnivMon + ] + ); + let tuple = ways_to_compute_intent(&AggIntent::Cardinality { cols: vec![0, 1] }); + assert!(!algorithms(&tuple).contains(&SketchAlgorithm::UnivMon)); +} + +/// Frequency moments offer a universal sketch without certification. +#[test] +fn frequency_statistics_keep_universal_choices() { + for intent in [ + AggIntent::FrequencyL2 { col: Some(0) }, + AggIntent::FrequencyEntropy { col: Some(0) }, + ] { + let choices = ways_to_compute_intent(&intent); + assert_eq!( + choices, + [HowToComputeAggregateMeasure::Summary( + SketchAlgorithm::UnivMon + )] + ); + } +} + +/// TopK keeps both declared heap substrates without choosing an implementation. +#[test] +fn topk_keeps_both_specialized_heap_choices() { + let choices = ways_to_compute_intent(&topk(10)); + assert_eq!( + algorithms(&choices), + vec![ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap + ] + ); +} + +/// Approximate count lists its exact accumulator first, beside the +/// summaries (the plain aggregate is the group's own expression). +#[test] +fn approximate_count_lists_its_accumulator_and_summaries() { + let choices = ways_to_compute_intent(&AggIntent::Count); + assert_eq!( + choices[0], + HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]) + ); + assert!(choices.contains(&HowToComputeAggregateMeasure::Summary( + SketchAlgorithm::UnivMon + ))); +} + +/// Count Sketch's signed counters give no total, so Pass 1 offers no Count +/// Sketch for a count. +#[test] +fn a_count_offers_no_count_sketch() { + assert!(!ways_to_compute_intent(&AggIntent::Count).contains( + &HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketch) + )); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/nested_merges.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/nested_merges.rs new file mode 100644 index 000000000..d89e4d733 --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/nested_merges.rs @@ -0,0 +1,87 @@ +use super::{memo_printout, selector, sum}; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::Memo; +use asap_types::summary_type::ExactKind; +use asap_types::test_support::memo_builders::{candidates_of, groups_where, query}; +use asap_types::workload::DataWorkload; + +/// `sum by (job) (sum_over_time(m[5m]))`, and likewise `min` of `min` and +/// `max` of `max`, gets a merged build: one exact accumulator of that kind +/// over the raw samples grouped by `job`, a candidate of the outer +/// aggregate's own accumulator state group, listing the inner aggregate's +/// group in `fused_groups`. +#[test] +fn nested_distributive_aggregate_merges_into_the_outer_state_group() { + for (intent, kind) in [ + (sum(), ExactKind::Sum), + (AggIntent::Min { col: None }, ExactKind::Min), + (AggIntent::Max { col: None }, ExactKind::Max), + ] { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![intent.clone()], + None, + ); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![intent], + None, + ); + query(&mut memo, by_job, None); + let memo = pass1_memo(memo, None).unwrap(); + let outer_state = groups_where( + &memo, + |e| matches!(e, Operator::ASAP(ASAPOp::SummaryAgg { child, summary_type: FieldDataType::ExactAggregate(state, _), .. }) if *child == per_series && *state == kind), + ); + let [outer_state] = outer_state[..] else { + panic!("{kind:?}: {}", memo_printout(&memo)) + }; + let builds = candidates_of(&memo, outer_state); + assert_eq!(builds.len(), 2, "{kind:?}: {}", memo_printout(&memo)); + let merged = &builds[1]; + assert_eq!(merged.expression.operator.children(), vec![&rows]); + assert_eq!(merged.fused_groups, vec![per_series]); + } +} + +/// The merge needs the same distributive aggregate twice: a `sum` over a +/// `max`, or a `min` over a `max`, gets none. +#[test] +fn nested_merge_needs_the_same_distributive_aggregate() { + for outer in [sum(), AggIntent::Min { col: None }] { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![AggIntent::Max { col: None }], + None, + ); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![outer], + None, + ); + query(&mut memo, by_job, None); + let memo = pass1_memo(memo, None).unwrap(); + assert!(memo + .groups() + .iter() + .flat_map(|g| &g.candidates) + .all(|c| c.fused_groups.is_empty())); + } +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/running_example.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/running_example.rs new file mode 100644 index 000000000..b22c9dadd --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/running_example.rs @@ -0,0 +1,102 @@ +use super::{memo_printout, quantile, selector, state_group}; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::{BinaryOpKind, Reduction}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::test_support::memo_builders::{candidates_of, group, query}; +use asap_types::workload::DataWorkload; + +/// The doc's running example: query A reads p95 of `latency`, query B +/// reads p99 / p50 of it. KLL state group holds one build per θ of KLL's +/// list, DDSketch's state group likewise; the p95, p99 and p50 groups each +/// read either state through their own readout groups. The metric rule +/// (B's relative ratio cannot read KLL's rank error) drops edges in Pass 2, +/// so Pass 1 offers both algorithms to every quantile group. +#[test] +fn running_example_gives_one_build_per_theta_and_separate_readouts() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = selector(&mut memo, TimeRangeKind::Instant, "latency", &[]); + let g3 = aggregate_group( + &mut memo, + g1, + Reduction::by(vec![]), + vec![quantile(0.95)], + None, + ); + let g4 = aggregate_group( + &mut memo, + g1, + Reduction::by(vec![]), + vec![quantile(0.99)], + None, + ); + let g5 = aggregate_group( + &mut memo, + g1, + Reduction::by(vec![]), + vec![quantile(0.5)], + None, + ); + let g6 = group( + &mut memo, + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: g4, + rhs: g5, + }), + ); + query(&mut memo, g3, None); + query(&mut memo, g6, None); + let memo = pass1_memo(memo, None).unwrap(); + println!("{}", memo_printout(&memo)); + let (kll, _) = state_group(&memo, SketchAlgorithm::Kll); + let (ddsketch, _) = state_group(&memo, SketchAlgorithm::DDSketch); + for (state, algorithm) in [ + (kll, SketchAlgorithm::Kll), + (ddsketch, SketchAlgorithm::DDSketch), + ] { + let thetas: Vec = candidates_of(&memo, state) + .iter() + .map(|candidate| candidate.logical_properties.parameters().unwrap().clone()) + .collect(); + assert_eq!(thetas, parameter_list(algorithm)); + // θ is a property, never an operator argument: every build has the + // group's own expression. + assert!(candidates_of(&memo, state).iter().all(|candidate| { + candidate.expression.operator == memo.groups()[state.0 as usize].expression + })); + } + // Each quantile group: its exact aggregate and one readout per algorithm, + // each through a readout group of its own over the state group. + for (result, q) in [(g3, 0.95), (g4, 0.99), (g5, 0.5)] { + let readouts: Vec = candidates_of(&memo, result)[1..] + .iter() + .map(|candidate| *candidate.expression.operator.children()[0]) + .collect(); + assert_eq!(readouts.len(), 2, "G{}", result.0); + for (readout, state) in readouts.iter().zip([kll, ddsketch]) { + assert_eq!( + memo.groups()[readout.0 as usize].expression, + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q }, + }) + ); + assert!(candidates_of(&memo, *readout)[0] + .logical_properties + .parameters() + .is_none()); + } + } + // The ratio's group keeps its one candidate, its own expression. + assert_eq!(candidates_of(&memo, g6).len(), 1); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/series_identity.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/series_identity.rs new file mode 100644 index 000000000..54913a895 --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/series_identity.rs @@ -0,0 +1,92 @@ +use std::time::Duration; + +use super::{memo_printout, quantile, selector}; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::schema::{DataType, PROMQL_SERIES_IDENTITY}; +use asap_types::ir::{NonASAPOp, Operator, ScalarExpr, TimeRangeKind}; +use asap_types::stages::Memo; +use asap_types::test_support::memo_builders::{candidates_of, group, query}; +use asap_types::test_support::memo_fixtures::scan; +use asap_types::workload::DataWorkload; + +/// `quantile_over_time(0.9, latency[5m])` over series whose identity is +/// realized as a column (as the planner does before Stage 0): each readout +/// candidate outputs the per-series aggregate's exact schema, the +/// `$promql_series_identity` column included, so every readout keeps the +/// series it answers for. +#[test] +fn per_series_readouts_keep_the_series_identity() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "latency", &["job"]); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![quantile(0.9)], + None, + ); + query(&mut memo, per_series, None); + let memo = pass1_memo(memo, None).unwrap(); + let output = &memo.groups()[per_series.0 as usize].group_properties.schema; + assert!(output.closed && output.has_promql_series_identity()); + let readouts = &candidates_of(&memo, per_series)[1..]; + assert_eq!(readouts.len(), 2, "{}", memo_printout(&memo)); + for readout in readouts { + let Operator::NonASAP(NonASAPOp::Project { child, cols, .. }) = + &readout.expression.operator + else { + panic!("a renaming Project: {:?}", readout.expression.operator) + }; + let estimate = &memo.groups()[child.0 as usize].group_properties.schema; + assert!(estimate.has_promql_series_identity()); + let identity = output + .fields + .iter() + .position(|field| field.name == PROMQL_SERIES_IDENTITY) + .unwrap(); + let read = estimate + .fields + .iter() + .position(|field| field.name == PROMQL_SERIES_IDENTITY) + .unwrap(); + assert_eq!(cols[identity].expr, ScalarExpr::Column(read)); + } +} + +/// Without a realized identity the per-series aggregate's schema is open: +/// its rows carry labels it does not list. A renaming `Project` would drop +/// them, so no readout is offered; the group keeps its own expression. +#[test] +fn an_open_per_series_aggregate_offers_no_renamed_readout() { + let mut memo = Memo::new(&DataWorkload::default()); + let scan = group( + &mut memo, + scan("latency", &[("job", DataType::Utf8)], vec![]), + ); + let rows = group( + &mut memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }), + ); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![quantile(0.9)], + None, + ); + query(&mut memo, per_series, None); + let memo = pass1_memo(memo, None).unwrap(); + assert!( + !memo.groups()[per_series.0 as usize] + .group_properties + .schema + .closed + ); + assert_eq!(candidates_of(&memo, per_series).len(), 1); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/summary_realization.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/summary_realization.rs new file mode 100644 index 000000000..c27a965dc --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/summary_realization.rs @@ -0,0 +1,132 @@ +use crate::computation_targets::ways_to_compute_intent; +use crate::summary_realization::{ + realize_keyed_additive_summary_input, summarised_input, summary_candidates, +}; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, FieldDataType}; +use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::{ + SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, +}; +use asap_types::test_support::memo_builders::{group, scan_group}; +use asap_types::test_support::memo_fixtures::scan; +use asap_types::workload::DataWorkload; + +/// A quantile outside [0, 1] is answered -Inf or +Inf: it gets no summary, +/// so no way besides the plain exact aggregate. +#[test] +fn a_quantile_outside_the_unit_interval_offers_no_summary() { + let choices = ways_to_compute_intent(&AggIntent::Quantile { col: None, q: 1.5 }); + assert!(choices.is_empty()); +} + +/// A heap sketch consumes every raw row, so the keyed input rule refuses +/// a TopK over a filtered count. +#[test] +fn keyed_heap_input_refuses_a_filtered_count() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = scan_group(&mut memo, "m", &["job"], vec![]); + let mut count_by_job = |filter: Option>| { + group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![2]), + measures: vec![AggIntent::Count], + output_names: vec![String::new()], + filters: vec![filter], + having: None, + child: rows, + }), + ) + }; + let unfiltered = count_by_job(None); + let value_above_one = Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Promql, + }); + let filtered = count_by_job(Some(value_above_one)); + let output = Reduction::Reduce(asap_types::ir::operator::arguments::GroupKeys::none()); + assert!(realize_keyed_additive_summary_input(&memo, &output, unfiltered).is_some()); + assert!(realize_keyed_additive_summary_input(&memo, &output, filtered).is_none()); +} + +/// A summary reads the intent's own input columns: one column by name, +/// none as the sample value, several as one hashed tuple. +#[test] +fn summarised_input_reads_the_intents_columns() { + // Columns [ts=0, value=1, job=2]. + let Operator::NonASAP(NonASAPOp::Scan { schema, .. }) = + scan("m", &[("job", DataType::Utf8)], vec![]) + else { + unreachable!("a scan") + }; + let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); + let cases = [ + (vec![2], column("job")), + (vec![1], column("value")), + (vec![], SummaryInputExpr::Column(ColumnRef::SampleValue)), + ( + vec![1, 2], + SummaryInputExpr::Tuple(vec![column("value"), column("job")]), + ), + ]; + for (cols, want) in cases { + let intent = AggIntent::Cardinality { cols: cols.clone() }; + assert_eq!( + summarised_input(&intent, &schema), + Some(want), + "wrong summarised input for cols={cols:?}" + ); + } +} + +/// Each approximable intent is offered the algorithms it was offered when +/// the list was written by hand, in the same order; a non-sketch intent +/// is offered none. +#[test] +fn summary_candidates_keep_the_hand_written_lists() { + use SketchAlgorithm as S; + let cardinality = |cols: Vec| AggIntent::Cardinality { cols }; + let cases = [ + ( + AggIntent::Quantile { col: None, q: 0.99 }, + vec![S::Kll, S::DDSketch], + ), + (cardinality(vec![]), vec![S::Hll, S::Kmv, S::UnivMon]), + (cardinality(vec![1]), vec![S::Hll, S::Kmv, S::UnivMon]), + (cardinality(vec![1, 2]), vec![S::Hll, S::Kmv]), + (AggIntent::FrequencyL2 { col: None }, vec![S::UnivMon]), + (AggIntent::FrequencyEntropy { col: None }, vec![S::UnivMon]), + ( + AggIntent::TopK { k: 10 }, + vec![S::CmsWithHeap, S::CountSketchWithHeap], + ), + (AggIntent::Count, vec![S::Cms, S::UnivMon]), + (AggIntent::Sum { col: None }, vec![]), + ]; + for (intent, want) in cases { + assert_eq!(summary_candidates(&intent), want, "{intent:?}"); + } +} + +/// An algorithm is offered for an intent exactly when its capability table +/// row answers the intent's statistic: Count-Min's row has no top-k readout, +/// so a top-k is offered only the heap variants, whose rows have one. +#[test] +fn an_algorithm_without_the_readout_is_not_offered() { + let top_k = AggIntent::TopK { k: 10 }; + let statistic = SketchStatistic::TopK { k: 10 }; + let answers = |algorithm| { + SummaryTypeCapability::of(&FieldDataType::Sketch(algorithm)).answers(&statistic) + }; + assert!(!answers(SketchAlgorithm::Cms)); + assert!(answers(SketchAlgorithm::CmsWithHeap)); + let offered = summary_candidates(&top_k); + assert!(!offered.contains(&SketchAlgorithm::Cms)); + assert!(offered.iter().all(|algorithm| answers(*algorithm))); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/targets.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/targets.rs new file mode 100644 index 000000000..e72f2bacd --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/targets.rs @@ -0,0 +1,116 @@ +use super::{quantile, selector, sum}; +use crate::computation_targets::computation_targets; +use crate::test_fixtures::aggregate_group; +use crate::{pass1_memo, PerComputationCandidateError}; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::{ExactKind, SummaryUpdate}; +use asap_types::test_support::memo_builders::{candidates_of, group, groups_where, query}; +use asap_types::workload::DataWorkload; + +/// Each measure of a multi-measure aggregate is its own target; a measure +/// whose only realization is its exact accumulator is still a target. +#[test] +fn multi_measure_aggregate_has_one_target_per_measure() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let both = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![sum(), quantile(0.9)], + None, + ); + query(&mut memo, both, None); + let targets = computation_targets(&memo, None).unwrap(); + let measures: Vec<(MemoGroupId, usize)> = targets + .iter() + .map(|target| (target.aggregate_group, target.measure_index)) + .collect(); + assert_eq!(measures, vec![(both, 0), (both, 1)]); +} + +/// An aggregate with no realization besides exact execution (no +/// accumulator, no summary: `stddev`) is not a target. +#[test] +fn aggregate_without_realization_is_not_a_target() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let stddev = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![AggIntent::StdDev { + col: None, + population: true, + }], + None, + ); + query(&mut memo, stddev, None); + assert!(computation_targets(&memo, None).unwrap().is_empty()); + let memo = pass1_memo(memo, None).unwrap(); + assert_eq!(candidates_of(&memo, stddev).len(), 1); +} + +/// An aggregate several queries read, or a scalar root reads twice, is one +/// target per measure: Pass 1 walks the groups. +#[test] +fn an_aggregate_read_by_several_roots_is_one_target() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let p = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + query(&mut memo, p, None); + query(&mut memo, p, None); + assert_eq!(computation_targets(&memo, None).unwrap().len(), 1); +} + +/// A memo whose groups already hold ASAP operators fails at the stage +/// boundary. +#[test] +fn asap_input_is_rejected() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let state = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: rows, + summary_type: FieldDataType::exact(ExactKind::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + ); + query(&mut memo, state, None); + assert!(matches!( + pass1_memo(memo, None), + Err(PerComputationCandidateError::ContainsAsap) + )); +} + +/// The summaries the deployment does not support are not offered. +#[test] +fn unsupported_summaries_are_not_offered() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &[]); + let p = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + query(&mut memo, p, None); + let memo = pass1_memo(memo, Some(&[])).unwrap(); + assert_eq!(candidates_of(&memo, p).len(), 1); + assert!(groups_where(&memo, |e| matches!(e, Operator::ASAP(_))).is_empty()); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs new file mode 100644 index 000000000..3df8a6c01 --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs @@ -0,0 +1,111 @@ +use std::time::Duration; + +use super::{quantile, topk}; +use crate::candidates::{add_measure_way, Pending, SingleMeasure}; +use crate::computation_targets::HowToComputeAggregateMeasure; +use crate::test_fixtures::aggregate_group; +use crate::PerComputationCandidateError; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::schema::DataType; +use asap_types::ir::{NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::SketchAlgorithm; +use asap_types::test_support::memo_builders::{group, scan_group}; +use asap_types::test_support::memo_fixtures::scan; +use asap_types::workload::DataWorkload; + +/// Offers `measure` by `way` to `result_group`, expecting `Unsupported`, and +/// checks the memo is unchanged: its groups, each group's candidates and +/// its hash-consing index (all in the memo's `Debug`). +fn assert_unsupported_way_adds_nothing( + memo: &mut Memo, + measure: &SingleMeasure<'_>, + way: HowToComputeAggregateMeasure, + result_group: MemoGroupId, +) { + let groups = memo.groups().len(); + let before = format!("{memo:?}"); + let added = add_measure_way(memo, Pending::new(memo), measure, &way, result_group, None); + assert!( + matches!(added, Err(PerComputationCandidateError::Unsupported(_))), + "{added:?}" + ); + assert_eq!(memo.groups().len(), groups); + assert_eq!(format!("{memo:?}"), before); +} + +/// A heap top-k whose estimate lacks a column of the result group (it +/// partitions `host` rows; the group ranks `job`): `ranked_as_topk` finds no +/// `job` in the pending estimate, so neither the heap's state group nor its +/// estimate group is added. +#[test] +fn a_heap_estimate_without_the_groups_columns_adds_nothing() { + let mut memo = Memo::new(&DataWorkload::default()); + let by_job = scan_group(&mut memo, "m", &["job"], vec![]); + let by_host = scan_group(&mut memo, "n", &["host"], vec![]); + let ranked = aggregate_group( + &mut memo, + by_job, + Reduction::by(vec![2]), + vec![topk(5)], + None, + ); + let reduction = Reduction::by(vec![2]); + let intent = topk(5); + let measure = SingleMeasure { + input: by_host, + unfiltered_input: by_host, + reduction: &reduction, + intent: &intent, + filter: None, + raw_data_samples_of: None, + }; + assert_unsupported_way_adds_nothing( + &mut memo, + &measure, + HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), + ranked, + ); +} + +/// A readout of an open per-series aggregate would need a renaming +/// `Project`, which drops the labels an open schema does not list: the +/// KLL state and readout groups are not added. +#[test] +fn a_readout_an_open_group_cannot_rename_adds_nothing() { + let mut memo = Memo::new(&DataWorkload::default()); + let scanned = group( + &mut memo, + scan("latency", &[("job", DataType::Utf8)], vec![]), + ); + let rows = group( + &mut memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scanned, + }), + ); + let intent = quantile(0.9); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![intent.clone()], + None, + ); + let measure = SingleMeasure { + input: rows, + unfiltered_input: rows, + reduction: &Reduction::PerEntity, + intent: &intent, + filter: None, + raw_data_samples_of: None, + }; + assert_unsupported_way_adds_nothing( + &mut memo, + &measure, + HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), + per_series, + ); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/update_weights.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/update_weights.rs new file mode 100644 index 000000000..5ab2e70dd --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/update_weights.rs @@ -0,0 +1,40 @@ +use super::{selector, state_group, sum}; +use crate::pass1_memo; +use crate::test_fixtures::aggregate_group; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::{ASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::Memo; +use asap_types::summary_type::{SketchAlgorithm, SummaryInputExpr}; +use asap_types::test_support::memo_builders::{groups_where, query}; +use asap_types::workload::DataWorkload; + +/// A Count-Min count adds one per row (bounded by the row count); a sum's +/// state reads the raw value column (bounded by the source's norm). +#[test] +fn update_weights_follow_the_measure() { + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let counts = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![AggIntent::Count], + None, + ); + query(&mut memo, counts, None); + let memo = pass1_memo(memo, None).unwrap(); + let (_, update) = state_group(&memo, SketchAlgorithm::Cms); + assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); + let mut memo = Memo::new(&DataWorkload::default()); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let sums = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); + query(&mut memo, sums, None); + let memo = pass1_memo(memo, None).unwrap(); + let state = groups_where( + &memo, + |e| matches!(e, Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) if input.weight == SummaryInputExpr::Column(ColumnRef::SampleValue)), + ); + assert_eq!(state.len(), 1); +} diff --git a/crates/stage1-pass2-reuse-candidates/Cargo.toml b/crates/stage1-pass2-reuse-candidates/Cargo.toml index 430c48f0f..88deb2865 100644 --- a/crates/stage1-pass2-reuse-candidates/Cargo.toml +++ b/crates/stage1-pass2-reuse-candidates/Cargo.toml @@ -3,12 +3,16 @@ name = "asap-stage1-pass2-reuse-candidates" version = "0.1.0" edition = "2021" -# #509 Stage 1, Pass 2: the reuse rules over memo groups and the logical +# Stage 1, Pass 2: the reuse rules over memo groups and the logical # edges (`asap_types::stages::LogicalASAPMemoCandidates`) Stage 2 expands. # Reads Pass 1's memo through `asap_types::stages`; depends on no other # stage crate. [dependencies] asap-types = { path = "../types" } -# The deployment's measured accuracy (`AccuracyEvidence`) Pass 2 reads. +# The accuracy rules Pass 2 applies: readout levels, the metric rule, +# alpha^log and the deployment's measured accuracy (`AccuracyEvidence`). asap-accuracy = { path = "../accuracy" } thiserror = "2" + +[dev-dependencies] +asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs new file mode 100644 index 000000000..d5bb16d28 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs @@ -0,0 +1,192 @@ +//! Steps 4 and 5, dead-candidate elimination (concepts doc, section 3): a +//! candidate that is not a query output is dead when it never had a +//! consumer, or when every edge to a consumer fails or reaches a dead +//! consumer; any candidate is dead when every edge on one of its inputs +//! fails or reads a dead producer. Elimination repeats until nothing +//! changes. Each dead candidate is listed with every root cause, one +//! shortest path per failing edge. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +use crate::edges::CandidateEdge; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, PruneCause, PrunedLogicalCandidate, RootPruneReason, +}; + +/// The dead candidates of `memo`, whose possible edges are `edges`, each +/// with its causes, in candidate order. +pub(crate) fn eliminate_dead_candidates( + memo: &Memo, + edges: &[CandidateEdge], +) -> Vec { + let graph = EdgeGraph::new(edges); + let outputs = memo.root_groups(); + let all: Vec = memo + .groups() + .iter() + .enumerate() + .flat_map(|(group, memo_group)| { + (0..memo_group.candidates.len()).map(move |index_in_group| CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }) + }) + .collect(); + // Per dead candidate, why, judged against those dead before it: cut + // off from an input (`Some(input)`), or without a live consumer. + let mut dead: BTreeMap> = BTreeMap::new(); + loop { + let newly_dead: Vec<(CandidateForMemoGroupId, Option)> = all + .iter() + .copied() + .filter(|candidate| !dead.contains_key(candidate)) + .filter_map(|candidate| { + if let Some(input) = graph.cut_off_input(memo, candidate, &dead) { + Some((candidate, Some(input))) + } else if !outputs.contains(&candidate.group) + && !graph.has_live_consumer(candidate, &dead) + { + Some((candidate, None)) + } else { + None + } + }) + .collect(); + if newly_dead.is_empty() { + break; + } + dead.extend(newly_dead); + } + dead.keys() + .map(|&candidate| PrunedLogicalCandidate { + candidate, + causes: graph.causes(candidate, &dead), + }) + .collect() +} + +/// `edges` by producer and by (consumer, input). +struct EdgeGraph<'a> { + by_producer: HashMap>, + by_consumer_input: HashMap<(CandidateForMemoGroupId, usize), Vec<&'a CandidateEdge>>, +} + +impl<'a> EdgeGraph<'a> { + fn new(edges: &'a [CandidateEdge]) -> Self { + let mut graph = Self { + by_producer: HashMap::new(), + by_consumer_input: HashMap::new(), + }; + for edge in edges { + graph + .by_producer + .entry(edge.producer) + .or_default() + .push(edge); + graph + .by_consumer_input + .entry((edge.consumer, edge.input)) + .or_default() + .push(edge); + } + graph + } + + /// Whether `candidate` has a satisfied edge into a consumer not in `dead`. + fn has_live_consumer( + &self, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> bool { + self.by_producer.get(&candidate).is_some_and(|edges| { + edges + .iter() + .any(|edge| edge.satisfies.is_ok() && !dead.contains_key(&edge.consumer)) + }) + } + + /// The first input of `candidate` with no satisfied edge from a producer + /// not in `dead`: every edge on it is cut (the metric rule, a missed + /// requirement) or reads an eliminated producer. + fn cut_off_input( + &self, + memo: &Memo, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> Option { + (0..memo + .candidate(candidate) + .expression + .operator + .children() + .len()) + .find(|&input| { + !self + .by_consumer_input + .get(&(candidate, input)) + .is_some_and(|edges| { + edges.iter().any(|edge| { + edge.satisfies.is_ok() && !dead.contains_key(&edge.producer) + }) + }) + }) + } + + /// Every root cause of dead `candidate`, breadth first, so each failing + /// edge is reached by a shortest path: a candidate cut off from an + /// input (`dead`) is traced along that input's edges, through + /// eliminated producers; any other through its consumer edges and + /// eliminated consumers, ending in `NoConsumer` at one that never had a + /// consumer. + fn causes( + &self, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> Vec { + let mut causes = Vec::new(); + let mut visited = BTreeSet::from([candidate]); + let mut queue = VecDeque::from([(candidate, Vec::new())]); + while let Some((at, path)) = queue.pop_front() { + // A candidate cut off from an input is traced to its producers, + // any other to its consumers. + let edges = match dead[&at] { + Some(input) => self + .by_consumer_input + .get(&(at, input)) + .map_or(&[][..], Vec::as_slice), + None => match self.by_producer.get(&at) { + Some(edges) => edges.as_slice(), + None => { + causes.push(PruneCause { + through_pruned_candidates: path, + root: RootPruneReason::NoConsumer, + }); + continue; + } + }, + }; + for edge in edges { + match &edge.satisfies { + Err(reason) => causes.push(PruneCause { + through_pruned_candidates: path.clone(), + root: reason.clone(), + }), + // A satisfied edge of a dead candidate leads to a dead + // consumer or producer. + Ok(()) => { + let next = match dead[&at] { + Some(_) => edge.producer, + None => edge.consumer, + }; + if visited.insert(next) { + let mut further = path.clone(); + further.push(next); + queue.push_back((next, further)); + } + } + } + } + } + causes + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/edges.rs b/crates/stage1-pass2-reuse-candidates/src/edges.rs new file mode 100644 index 000000000..f4095894e --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/edges.rs @@ -0,0 +1,91 @@ +//! Steps 4 to 6, the logical edges: every possible $(u, c, i)$ of the memo with +//! $\alpha^{\log}$, which dead-candidate elimination reads and whose +//! satisfied edges between survivors are Pass 2's output. + +use std::collections::BTreeMap; + +use asap_accuracy::{AccuracyEvidence, ErrorMetricsOfMemo}; +use asap_types::stages::{ + CandidateForMemoGroupId, EvaluationGrid, Memo, MemoGroupId, RootPruneReason, +}; + +use crate::evaluation_grids::chain_off_the_evaluation_grid; + +/// One possible $(u, c, i)$: `producer` is a candidate of the group input +/// `input` of `consumer` reads, and whether it satisfies that input's +/// logical requirement (`ErrorMetricsOfMemo::edge_meets_logical_requirements`). +pub(crate) struct CandidateEdge { + pub(crate) producer: CandidateForMemoGroupId, + pub(crate) consumer: CandidateForMemoGroupId, + pub(crate) input: usize, + pub(crate) satisfies: Result<(), RootPruneReason>, +} + +/// Every $(u, c, i)$ of `memo` (`Memo::producers`), grouped by consumer and +/// input, producers in group order; `error_metrics` is +/// `ErrorMetricsOfMemo::of` `memo`. An edge into a readout of +/// `readouts_without_per_value_failure_probability` fails with +/// `ValuesPerEvaluationUnknown`. An edge from a chain whose windows do not +/// end at the consuming group's evaluation times +/// ([`chain_off_the_evaluation_grid`], from `evaluation_grid_of_group`) +/// fails with `ChainWidthDoesNotDivideEvaluationGrid`: a chain of another +/// reader's width does not end at this reader's evaluations. +pub(crate) fn candidate_edges( + memo: &Memo, + error_metrics: &ErrorMetricsOfMemo, + evaluation_grid_of_group: &[Option], + readouts_without_per_value_failure_probability: &BTreeMap, + evidence: &dyn AccuracyEvidence, +) -> Vec { + let mut edges = Vec::new(); + for (group, memo_group) in memo.groups().iter().enumerate() { + let group = MemoGroupId(group as u32); + for (index_in_group, candidate) in memo_group.candidates.iter().enumerate() { + let consumer = CandidateForMemoGroupId { + group, + index_in_group, + }; + for (input, &input_group) in candidate + .expression + .operator + .children() + .into_iter() + .enumerate() + { + let chain_off_the_grid = chain_off_the_evaluation_grid( + memo, + evaluation_grid_of_group, + group, + input_group, + ); + for producer in memo.producers(consumer, input) { + edges.push(CandidateEdge { + producer, + consumer, + input, + satisfies: match ( + chain_off_the_grid, + readouts_without_per_value_failure_probability.get(&consumer), + ) { + (Some((tumbling_window_width_ms, evaluation_period_ms)), _) => { + Err(RootPruneReason::ChainWidthDoesNotDivideEvaluationGrid { + consumer, + input, + tumbling_window_width_ms, + evaluation_period_ms, + }) + } + (None, Some(&query)) => { + Err(RootPruneReason::ValuesPerEvaluationUnknown { consumer, query }) + } + (None, None) => error_metrics.edge_meets_logical_requirements( + memo, producer, consumer, input, evidence, + ), + }, + }); + } + } + } + } + edges +} diff --git a/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs new file mode 100644 index 000000000..1cbb9e09f --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs @@ -0,0 +1,147 @@ +//! Steps 4 to 6: the evaluation grid of each group, per query reaching +//! it: the times a group's result is evaluated at, which a chain's width +//! must divide. + +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; + +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{EvaluationGrid, Memo, MemoGroupId}; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; + +/// Per group, by group id: per query reaching it, the period in ms of the +/// grid the query evaluates the group's result on (phase 0), `None` when +/// that period is unknown. Walking down from each query's root, which is on +/// the query's own grid ([`query_evaluation_period_ms`]), a group is on its +/// consumer's grid, except below a PromQL subquery: `[10m:30s]` +/// evaluates `` at the multiples of 30 s whatever the outer grid, and +/// `[10m:]` at the server's default interval, unknown to the planner. +/// A group a query reaches through several paths is on the gcd of their +/// grids (unknown when one is). Every candidate's inputs are walked, so a +/// rollup's finer group is on the grids of the queries rolled up. +pub(crate) fn evaluation_period_ms_per_query( + memo: &Memo, + entries: &[QueryWorkloadEntry], +) -> Vec>> { + let mut periods_ms: Vec>> = + vec![BTreeMap::new(); memo.groups().len()]; + let mut pending = Vec::new(); + for (query, memo_query) in memo.queries().iter().enumerate() { + let period_ms = query_evaluation_period_ms(entries, query); + for &root in memo_query.root.operator_refs() { + if merge(&mut periods_ms[root.0 as usize], query, period_ms) { + pending.push(root); + } + } + } + while let Some(group) = pending.pop() { + let periods_here_ms = periods_ms[group.0 as usize].clone(); + for candidate in &memo.groups()[group.0 as usize].candidates { + let operator = &candidate.expression.operator; + let subquery_resolution_ms = match operator { + Operator::NonASAP(NonASAPOp::PromqlSubquery { resolution, .. }) => Some( + resolution + .map(|resolution| resolution.as_millis() as u64) + .filter(|&ms| ms > 0), + ), + _ => None, + }; + for &child in operator.children() { + for (&query, &period_ms) in &periods_here_ms { + let below_ms = subquery_resolution_ms.unwrap_or(period_ms); + if merge(&mut periods_ms[child.0 as usize], query, below_ms) { + pending.push(child); + } + } + } + } + } + periods_ms +} + +/// Per group: the grid its result is evaluated on, the gcd of the +/// evaluation periods of the queries reaching it +/// (`evaluation_period_ms_per_query`) at phase 0; `None` when no query +/// reaches it or one evaluates it on no known grid. +pub(crate) fn evaluation_grids( + evaluation_period_ms_per_query: &[BTreeMap>], +) -> Vec> { + evaluation_period_ms_per_query + .iter() + .map(|periods_ms| { + let mut period_ms = 0; + for &query_period_ms in periods_ms.values() { + period_ms = greatest_common_divisor(period_ms, query_period_ms?); + } + (period_ms > 0).then_some(EvaluationGrid { + period_ms, + phase_ms: 0, + }) + }) + .collect() +} + +/// When `input_group` is a chain `SummaryTumblingWindowChain[w]` read by +/// `group`, whose result depends on the evaluation time, and $w$ does not +/// divide `group`'s evaluation period (`evaluation_grid_of_group`, from +/// [`evaluation_grids`]) or that period is unknown: the chain's width and +/// the period in ms. Its windows do not end at `group`'s evaluation times. +pub(crate) fn chain_off_the_evaluation_grid( + memo: &Memo, + evaluation_grid_of_group: &[Option], + group: MemoGroupId, + input_group: MemoGroupId, +) -> Option<(u64, Option)> { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) = memo.groups()[input_group.0 as usize].expression + else { + return None; + }; + if !memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time + { + return None; + } + let evaluation_period_ms = + evaluation_grid_of_group[group.0 as usize].map(|grid| grid.period_ms); + (!evaluation_period_ms + .is_some_and(|period_ms| period_ms.is_multiple_of(tumbling_window_width_ms))) + .then_some((tumbling_window_width_ms, evaluation_period_ms)) +} + +/// The evaluation period in ms of query `query` (`entries[query]`'s +/// `QueryRecurrence::evaluation_grid_ms`); `None` when it has no known +/// grid. +fn query_evaluation_period_ms(entries: &[QueryWorkloadEntry], query: usize) -> Option { + let entry = entries.get(query)?; + entry.recurrence.evaluation_grid_ms(entry.range_evaluation) +} + +/// Adds `query`'s evaluation period `period_ms` to a group's periods: the +/// gcd with a known one, unknown when either is; whether they changed. +fn merge( + periods_ms: &mut BTreeMap>, + query: usize, + period_ms: Option, +) -> bool { + match periods_ms.entry(query) { + Entry::Vacant(entry) => { + entry.insert(period_ms); + true + } + Entry::Occupied(mut entry) => { + let merged = match (*entry.get(), period_ms) { + (Some(known_ms), Some(period_ms)) => { + Some(greatest_common_divisor(known_ms, period_ms)) + } + _ => None, + }; + let changed = merged != *entry.get(); + entry.insert(merged); + changed + } + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/lib.rs b/crates/stage1-pass2-reuse-candidates/src/lib.rs index db7860446..7331be4b0 100644 --- a/crates/stage1-pass2-reuse-candidates/src/lib.rs +++ b/crates/stage1-pass2-reuse-candidates/src/lib.rs @@ -1,4 +1,4 @@ -//! `asap-stage1-pass2-reuse-candidates` — #509 Stage 1, Pass 2: the reuse +//! `asap-stage1-pass2-reuse-candidates` — Stage 1, Pass 2: the reuse //! rules over memo groups, the readouts' required accuracy, dead-candidate //! elimination and the logical edges. Its entry, //! [`generate_logical_asap_memo_candidates`], extends Pass 1's memo into @@ -7,27 +7,67 @@ //! count reading one state. //! //! It depends on `asap-types`, where its input and output are, and on -//! `asap-accuracy` for $\alpha^{\log}$. It depends on no other stage crate, -//! and the planner facade calls it after Pass 1. +//! `asap-accuracy` for the readout levels and $\alpha^{\log}$. It depends on +//! no other stage crate, and the planner facade calls it after Pass 1. +/// Steps 4 and 5: dead-candidate elimination and its root causes. +mod dead_candidates; +/// Steps 4 to 6: the possible edges and their $\alpha^{\log}$. +mod edges; +/// Steps 4 to 6: the evaluation grid of each group, which a chain's width +/// must divide. +mod evaluation_grids; +/// Step 3: readout requirements. +mod readout_requirements; /// Steps 1, 2 and 5: the rules that add groups and candidates. mod rules; +#[cfg(test)] +mod test_fixtures; +#[cfg(test)] +mod tests; -use asap_accuracy::AccuracyEvidence; -use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; +use std::collections::BTreeSet; + +use asap_accuracy::{AccuracyEvidence, ErrorMetricsOfMemo}; +use asap_types::stages::{ + CandidateError, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalEdge, Memo, + MemoGroupError, PrunedLogicalCandidate, +}; use asap_types::workload::QueryWorkloadEntry; -pub use rules::Pass2RuleError; -use rules::{rollup, topk}; +use dead_candidates::eliminate_dead_candidates; +use edges::candidate_edges; +use evaluation_grids::{evaluation_grids, evaluation_period_ms_per_query}; +use readout_requirements::set_readout_requirements; +use rules::{chains, rollup, topk}; + +/// Why Pass 2 failed on a memo: a rule built an expression the node rules +/// reject (a bug in the rule), or the memo's reads form a cycle. +#[derive(Debug, thiserror::Error)] +pub enum Pass2RuleError { + /// A group, or one of its candidates, reads itself through other groups. + #[error("memo dependencies contain a cycle")] + CyclicDependencies, + /// A rule built a group expression the node rules reject. + #[error("a rule built a group that breaks the node rules: {0}")] + Group(#[from] MemoGroupError), + /// A rule built a candidate expression the node rules reject, or gave + /// it θ its group's output cannot have. + #[error("a rule built a candidate that breaks the node rules: {0}")] + Candidate(CandidateError), +} /// Stage 1, Pass 2 (concepts doc, section 3, "Stage 1, Pass 2"), in order: /// 1. rollup from the finest common group; /// 2. top-k reuse; /// 3. reach, caps and readout requirements; -/// 4. dead-candidate elimination; -/// 5. chain groups for the surviving state groups, then dead-candidate -/// elimination again over the memo with them; +/// 4. dead-candidate elimination over the logical edges; +/// 5. chain groups for the surviving state groups, then the edges and +/// dead-candidate elimination again over the memo with them (a merge +/// whose θ serves no consumer); /// 6. the logical edges between the surviving candidates, with -/// $\alpha^{\log}$ from `evidence` or the theoretical guarantee. +/// $\alpha^{\log}$ from `evidence` or the theoretical guarantee, and the +/// groups each query may read only exactly +/// (`ErrorMetricsOfMemo::exact_only_groups_of_query`). /// /// `entries[i]` is query `i`'s workload entry (its evaluation grid). pub fn generate_logical_asap_memo_candidates( @@ -38,9 +78,67 @@ pub fn generate_logical_asap_memo_candidates( let mut memo = pass1; rollup::add_rollup_candidates(&mut memo)?; topk::add_topk_candidates(&mut memo)?; - let _ = (memo, entries, evidence); - todo!( - "#1207 PR 4: steps 3-6: readout requirements, dead-candidate elimination, \ - chain groups (`rules::chains::add_chain_groups`) and the logical edges" - ) + let topological_order = |memo: &Memo| { + memo.topological_order() + .ok_or(Pass2RuleError::CyclicDependencies) + }; + // Readout copies add candidates, not groups: one order serves steps 3 + // and 4. + let order = topological_order(&memo)?; + let readout_requirements = set_readout_requirements(&mut memo, &order)?; + let candidates_of = |pruned: &[PrunedLogicalCandidate]| -> BTreeSet { + pruned.iter().map(|pruned| pruned.candidate).collect() + }; + let evaluation_period_ms = evaluation_period_ms_per_query(&memo, entries); + let before_chains = eliminate_dead_candidates( + &memo, + &candidate_edges( + &memo, + &ErrorMetricsOfMemo::of(&memo, &order), + &evaluation_grids(&evaluation_period_ms), + &readout_requirements.readouts_without_per_value_failure_probability, + evidence, + ), + ); + chains::add_chain_groups( + &mut memo, + &evaluation_period_ms, + &candidates_of(&before_chains), + )?; + let order = topological_order(&memo)?; + let evaluation_grid_of_group = + evaluation_grids(&evaluation_period_ms_per_query(&memo, entries)); + let error_metrics = ErrorMetricsOfMemo::of(&memo, &order); + let candidate_edges = candidate_edges( + &memo, + &error_metrics, + &evaluation_grid_of_group, + &readout_requirements.readouts_without_per_value_failure_probability, + evidence, + ); + let pruned = eliminate_dead_candidates(&memo, &candidate_edges); + let dead = candidates_of(&pruned); + let edges = candidate_edges + .into_iter() + .filter(|edge| !dead.contains(&edge.producer) && !dead.contains(&edge.consumer)) + .map(|edge| LogicalEdge { + producer_candidate_id: edge.producer, + consumer_candidate_id: edge.consumer, + input_index_of_consumer: edge.input, + satisfies_logical_requirement: edge.satisfies.is_ok(), + }) + .collect(); + let exact_only_groups_of_query = error_metrics.exact_only_groups_of_query( + &memo, + &order, + &dead, + &readout_requirements.exact_only_readout_groups_of_query, + ); + Ok(LogicalASAPMemoCandidates { + memo, + edges, + pruned, + evaluation_grid_of_group, + exact_only_groups_of_query, + }) } diff --git a/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs new file mode 100644 index 000000000..83ece4d84 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs @@ -0,0 +1,90 @@ +//! Step 3 (concepts doc, section 3, "Reach, caps and readout requirements"; +//! section 4, "Readout requirements"): writes each readout's required +//! accuracy, from the levels `asap_accuracy::readout_levels` gives it, on +//! the readout and its copies. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::Pass2RuleError; +use asap_accuracy::{readout_levels, ErrorMetricsOfMemo}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalRequirements, Memo, MemoGroupId, RequiredAccuracy, +}; + +/// What step 3 leaves for the later steps beside the requirements it +/// writes. +pub(crate) struct ReadoutRequirementsOutcome { + /// The readouts no query gives a level while one reaching them has a + /// failure probability without a per-value one, each with that query: + /// Pass 2 prunes them (`RootPruneReason::ValuesPerEvaluationUnknown`). + pub(crate) readouts_without_per_value_failure_probability: + BTreeMap, + /// Per such query, the readout groups it reaches without a per-value + /// failure probability, which it may read only exactly (another + /// query's level says nothing of its own δ). + pub(crate) exact_only_readout_groups_of_query: BTreeMap>, +} + +/// Gives every sketch readout (`SummaryEstimate` over a sketch state) its +/// required accuracy, one readout candidate per requirement level +/// ([`readout_levels`]): Pass 1's candidate takes the first level and a +/// copy is added per further level. A readout without a level keeps no +/// requirement. `order` is the memo's topological order. +pub(crate) fn set_readout_requirements( + memo: &mut Memo, + order: &[MemoGroupId], +) -> Result { + let mut outcome = ReadoutRequirementsOutcome { + readouts_without_per_value_failure_probability: BTreeMap::new(), + exact_only_readout_groups_of_query: BTreeMap::new(), + }; + let error_metrics = ErrorMetricsOfMemo::of(memo, order); + for (readout, required) in readout_levels(memo, &error_metrics, order) { + for &query in &required.queries_without_per_value_failure_probability { + outcome + .exact_only_readout_groups_of_query + .entry(query) + .or_default() + .insert(readout.group); + } + match required + .queries_without_per_value_failure_probability + .first() + { + Some(&query) if required.levels.is_empty() => { + outcome + .readouts_without_per_value_failure_probability + .insert(readout, query); + } + _ => write_levels(memo, readout, &required.levels)?, + } + } + Ok(outcome) +} + +/// Writes `levels` on `readout`: the first on it, each further one on a +/// copy of it added to its group with that level. No level leaves it +/// without a requirement. +fn write_levels( + memo: &mut Memo, + readout: CandidateForMemoGroupId, + levels: &[RequiredAccuracy], +) -> Result<(), Pass2RuleError> { + let requirement = |level: RequiredAccuracy| { + vec![LogicalRequirements { + required_accuracy: Some(level), + }] + }; + let Some((&first, further)) = levels.split_first() else { + return Ok(()); + }; + let unset = memo.candidate(readout).clone(); + memo.set_candidate_input_requirements(readout, requirement(first)); + for &level in further { + let mut copy = unset.clone(); + copy.input_requirements = requirement(level); + memo.add_candidate(readout.group, copy) + .map_err(Pass2RuleError::Candidate)?; + } + Ok(()) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules.rs b/crates/stage1-pass2-reuse-candidates/src/rules.rs deleted file mode 100644 index 039b01b89..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/rules.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Pass 2's rules over groups (concepts doc, section 3, "Stage 1, Pass 2"): -//! each reads the memo and adds groups (`Memo::find_or_add_group`) or -//! candidates (`Memo::add_candidate`); none copies a consumer. -pub(crate) mod chains; -pub(crate) mod rollup; -pub(crate) mod topk; - -/// Why a Pass 2 rule failed on a memo. -#[derive(Debug, thiserror::Error)] -pub enum Pass2RuleError { - #[error("memo contains an out-of-range group reference")] - InvalidGroupReference, - #[error("memo group capacity exceeds the group identifier range")] - TooManyGroups, - #[error("memo dependencies contain a cycle")] - CyclicDependencies, -} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/candidates.rs b/crates/stage1-pass2-reuse-candidates/src/rules/candidates.rs new file mode 100644 index 000000000..f5e8f199e --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/candidates.rs @@ -0,0 +1,80 @@ +//! How the rules add candidates: a candidate whose output the memo +//! accepts as its group's, one per θ of the states an operator passes on, +//! and the group of an expression computing itself. + +use crate::Pass2RuleError; +use asap_types::ir::Operator; +use asap_types::stages::{ + CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::SketchParams; + +/// Adds the candidate `operator` with $\boldsymbol\theta$ `parameters` to +/// `group`, its inputs without a requirement (step 3 sets the readouts'). +/// `None` when its output is not the group's +/// (`CandidateError::OutputDiffersFromGroup`): the rules match shapes and +/// let the memo decide whether a match computes the group's result. Any +/// other rejection is a bug in the rule and an error. +pub(crate) fn add_candidate_if_output_matches( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, + origin: OriginOfCandidateForMemoGroup, +) -> Result, Pass2RuleError> { + match memo.add_candidate( + group, + CandidateForMemoGroup::new(operator, parameters, origin, Vec::new()), + ) { + Ok(id) => Ok(Some(id)), + Err(CandidateError::OutputDiffersFromGroup { .. }) => Ok(None), + Err(error) => Err(Pass2RuleError::Candidate(error)), + } +} + +/// The distinct θ of `candidates`, in order. +pub(crate) fn distinct_parameters<'a>( + candidates: impl IntoIterator, +) -> Vec> { + let mut parameters = Vec::new(); + for candidate in candidates { + let theta = candidate.logical_properties.parameters().cloned(); + if !parameters.contains(&theta) { + parameters.push(theta); + } + } + parameters +} + +/// Adds `operator`, which passes on the states or rows of its one input +/// group (a `SummaryMerge`, or a `Filter`, `TimeRange` or `TimeShift`), to +/// `group`: one candidate per θ among the input group's candidates (concepts +/// doc, section 2: such a candidate carries the θ of the states it reads), +/// one without a θ over rows or exact states. +pub(crate) fn add_candidates_per_input_parameters( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + origin: OriginOfCandidateForMemoGroup, +) -> Result<(), Pass2RuleError> { + let input = *operator.children()[0]; + for theta in distinct_parameters(&memo.groups()[input.0 as usize].candidates) { + add_candidate_if_output_matches(memo, group, operator.clone(), theta, origin.clone())?; + } + Ok(()) +} + +/// The group of `operator`, with its own expression as its candidates +/// ([`add_candidates_per_input_parameters`]): a group a rule adds between +/// others, a compensating `Filter`, a `TimeRange` or `TimeShift` over a +/// chain. +pub(crate) fn add_group_computing_itself( + memo: &mut Memo, + operator: Operator, + origin: OriginOfCandidateForMemoGroup, +) -> Result { + let group = memo.find_or_add_group(operator.clone())?; + add_candidates_per_input_parameters(memo, group, operator, origin)?; + Ok(group) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs index 3b64862f2..9784f06af 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs @@ -3,24 +3,256 @@ //! over the `TimeRange` (and `TimeShift`s) over it as a candidate of the //! summary state group. -use std::collections::BTreeSet; - -use crate::rules::Pass2RuleError; -use asap_types::stages::{CandidateForMemoGroupId, Memo}; -use asap_types::workload::QueryWorkloadEntry; - -/// For each summary state group with a build that survived dead-candidate -/// elimination (not in `pruned`), a chain group and a merge candidate per -/// tumbling width $w$ from the gcd closure of the evaluation grids, ranges -/// and offsets of the queries that reach it and repeat on a known grid. -/// `entries[q]` is query `q`'s workload entry. -// Step 5 runs after dead-candidate elimination, which #1207 PR 4 adds. -#[allow(dead_code)] +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rules::candidates::{ + add_candidate_if_output_matches, add_candidates_per_input_parameters, + add_group_computing_itself, distinct_parameters, +}; +use crate::Pass2RuleError; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, TimeShift}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, Predicate, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::{SketchParams, SummaryTypeCapability, SummaryUpdate}; +use asap_types::workload::greatest_common_divisor; + +/// For each summary state group S = `SummaryAgg(TimeRange(W, +/// TimeShift*(X)))` of a summary type that merges, with a build that +/// survived dead-candidate elimination (`pruned`), and each width $w$: the +/// chain group $G_w$ = `SummaryTumblingWindowChain[w](X)` (the shifts +/// stripped, so ranges and offsets over X share it) with one build per θ of +/// S's surviving builds, the shifts and the `TimeRange` re-wrapped around +/// it, and the candidates `SummaryMerge` over the outermost of them in S, +/// each of these one per θ of the group it reads (a chain shared by several +/// state groups has the builds of all of them). +/// +/// The widths are the gcd closure of $\gcd(g_q, W, o_1, \ldots)$ over the +/// queries reaching S whose evaluation period at S, $g_q$, is known +/// (`evaluation_period_ms_per_query`: below a subquery, its resolution), +/// with the offsets $o_j$ on S's path; no query with a period, no chain. +/// The edges from a chain whose width does not divide the evaluation grid +/// of the group reading it (the gcd over all its readers) fail with +/// `ChainWidthDoesNotDivideEvaluationGrid` (`candidate_edges`), so the +/// merge over it is pruned: a 15 s chain never serves a reader every 10 s. +/// Under `@ T` S adds no width: once every unanchored state has added its +/// chains, it reads those whose width divides W, the offsets and T, in any +/// memo order. Under `@ end()` or `@ start()` (a `TimeShift` the front end +/// puts above the subtree it anchors) no chain is read. pub(crate) fn add_chain_groups( memo: &mut Memo, - entries: &[QueryWorkloadEntry], + evaluation_period_ms_per_query: &[BTreeMap>], pruned: &BTreeSet, ) -> Result<(), Pass2RuleError> { - let _ = (memo, entries, pruned); - todo!("#1207 PR 4: step 5, chain groups and the merges over them") + // The chainable states with the θ of their surviving builds, the + // unanchored ones first: an anchored state reads only the chains they + // create. + let mut states: Vec<(MemoGroupId, ChainPath, Vec>)> = Vec::new(); + for (index, memo_group) in memo.groups().iter().enumerate() { + let state = MemoGroupId(index as u32); + let Some(path) = chain_path(memo, state) else { + continue; + }; + let parameters = distinct_parameters( + memo_group + .candidates + .iter() + .enumerate() + .filter(|&(index_in_group, candidate)| { + let survived = !pruned.contains(&CandidateForMemoGroupId { + group: state, + index_in_group, + }); + survived && candidate.expression.operator == memo_group.expression + }) + .map(|(_, candidate)| candidate), + ); + if !parameters.is_empty() { + states.push((state, path, parameters)); + } + } + states.sort_by_key(|(_, path, _)| path.anchored_at_ms.is_some()); + // First every chain group with its builds, then the groups reading + // them, so a chain shared by state groups offers each every θ it has. + let mut chains_of_states: Vec<(MemoGroupId, ChainPath, Vec)> = Vec::new(); + for (state, path, parameters) in states { + let divisor = path.shifts.iter().fold(path.window_ms, |gcd, shift| { + greatest_common_divisor(gcd, shift.offset_ms.unsigned_abs()) + }); + let widths: BTreeSet = match path.anchored_at_ms { + Some(at_ms) => existing_chain_widths(memo, &path) + .into_iter() + .filter(|&width| divisor % width == 0 && at_ms.unsigned_abs() % width == 0) + .collect(), + None => gcd_closure( + evaluation_period_ms_per_query[state.0 as usize] + .values() + .filter_map(|&period_ms| Some(greatest_common_divisor(period_ms?, divisor))), + ), + }; + let mut chain_groups = Vec::new(); + for width in widths { + let chain = path.chain(width); + let chain_group = memo.find_or_add_group(chain.clone())?; + for theta in ¶meters { + add_candidate_if_output_matches( + memo, + chain_group, + chain.clone(), + theta.clone(), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + chain_groups.push(chain_group); + } + chains_of_states.push((state, path, chain_groups)); + } + for (state, path, chain_groups) in chains_of_states { + for chain_group in chain_groups { + let mut top = chain_group; + for shift in path.shifts.iter().rev() { + top = add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: *shift, + child: top, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + let windows = add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_millis(path.window_ms), + kind: TimeRangeKind::Range, + child: top, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + add_candidates_per_input_parameters( + memo, + state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + } + Ok(()) +} + +/// A state group's walk down to the rows its chain reads: +/// `SummaryAgg` → `TimeRange(W)` → `TimeShift`s → X, with the build's +/// arguments the chain keeps. +struct ChainPath { + summary_type: FieldDataType, + input: SummaryUpdate, + reduction: Reduction, + filter: Option>, + window_ms: u64, + /// Top down. + shifts: Vec, + /// T of an `@ T` among the shifts. + anchored_at_ms: Option, + rows: MemoGroupId, +} + +impl ChainPath { + /// `SummaryTumblingWindowChain[width](X)` with the build's arguments. + fn chain(&self, width: u64) -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: self.rows, + summary_type: self.summary_type.clone(), + input: self.input.clone(), + reduction: self.reduction.clone(), + filter: self.filter.clone(), + tumbling_window_width_ms: width, + }) + } +} + +/// S's [`ChainPath`]: a `SummaryAgg` of a summary type whose states merge +/// (its capability row), over a range `TimeRange(W)` with `W > 0`, over +/// `TimeShift`s with at most one `@ T` and no `@ end()` or `@ start()`. +fn chain_path(memo: &Memo, state: MemoGroupId) -> Option { + let expression = &memo.groups()[state.0 as usize].expression; + let Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + input, + reduction, + filter, + }) = expression + else { + return None; + }; + SummaryTypeCapability::of(summary_type).merge_inputs?; + let Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + child: below, + }) = &memo.groups()[child.0 as usize].expression + else { + return None; + }; + let window_ms = u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0)?; + let (mut shifts, mut anchored_at_ms, mut rows) = (Vec::new(), None, *below); + while let Operator::NonASAP(NonASAPOp::TimeShift { shift, child }) = + &memo.groups()[rows.0 as usize].expression + { + match shift.at { + None => {} + Some(AtModifier::Timestamp(at_ms)) if anchored_at_ms.is_none() => { + anchored_at_ms = Some(at_ms) + } + Some(_) => return None, + } + shifts.push(*shift); + rows = *child; + } + Some(ChainPath { + summary_type: summary_type.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: filter.clone(), + window_ms, + shifts, + anchored_at_ms, + rows, + }) +} + +/// The widths of the chain groups over `path`'s rows with its build's +/// arguments that the memo has already. +fn existing_chain_widths(memo: &Memo, path: &ChainPath) -> Vec { + memo.groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) if group.expression == path.chain(*tumbling_window_width_ms) => { + Some(*tumbling_window_width_ms) + } + _ => None, + }) + .collect() +} + +/// The gcd closure of `widths`: each, and the gcd of every two in it. +fn gcd_closure(widths: impl IntoIterator) -> BTreeSet { + let mut closure: BTreeSet = BTreeSet::new(); + for width in widths.into_iter().filter(|&width| width > 0) { + let gcds: Vec = closure + .iter() + .map(|&known| greatest_common_divisor(known, width)) + .collect(); + closure.insert(width); + closure.extend(gcds); + } + closure } diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/mod.rs b/crates/stage1-pass2-reuse-candidates/src/rules/mod.rs new file mode 100644 index 000000000..b04d231f0 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/mod.rs @@ -0,0 +1,8 @@ +//! Pass 2's rules over groups (concepts doc, section 3, "Stage 1, Pass 2"): +//! each reads the memo and adds groups (`Memo::find_or_add_group`) or +//! candidates (`Memo::add_candidate`, through `candidates`); none copies a +//! consumer, so every consumer of a group sees what a rule adds to it. +pub(crate) mod candidates; +pub(crate) mod chains; +pub(crate) mod rollup; +pub(crate) mod topk; diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs deleted file mode 100644 index b7676185a..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Step 1, rollup from the finest common group (concepts doc, section 3): -//! a coarse aggregate group $G_c$ computed from a finer group $G_f$ of the -//! same definition, and per cluster of equal definitions one finest common -//! group $G^*$ every member can roll up from. - -use crate::rules::Pass2RuleError; -use asap_types::stages::Memo; - -/// Adds, to each aggregate group $G_c$, a candidate computing it from each -/// finer group $G_f$ of its cluster, after synthesizing each cluster's -/// finest common group. A cluster is the summary state groups (`SummaryAgg` -/// of a summary type whose states merge across groups) or exact `Aggregate` -/// groups (of sums, counts, minima and maxima) with equal definitions apart -/// from their keys and equal interval constraints (the time window). -pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: step 1, rollup from the finest common group") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup/finest_common_group.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/finest_common_group.rs new file mode 100644 index 000000000..0f9f72ac9 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/finest_common_group.rs @@ -0,0 +1,114 @@ +//! A cluster's finest common group $G^*$, which every member can roll up +//! from. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rules::candidates::add_candidate_if_output_matches; +use crate::Pass2RuleError; +use asap_types::ir::operator::arguments::SourceColumn; +use asap_types::ir::operator::Reduction; +use asap_types::ir::properties::selection::Constraint; +use asap_types::ir::scalar::ScalarValue; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{ + listed_build_parameters, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; + +use super::member::{column_of, Member}; + +/// The cluster's finest common group $G^*$: its selection is, per column, +/// the union of the members' value-set constraints (none where a member +/// has none); its keys are every member's keys plus each column on which a +/// member selects fewer rows than $G^*$. It is a member when one has that +/// selection and those keys; otherwise it is built over the rows of a +/// member whose selection is the union, with one build per listed θ. +/// `None` when no member's rows have the union selection (a new scan of +/// the union is not built) or a key is not a column of those rows. +pub(super) fn finest_common_group( + memo: &mut Memo, + cluster: &[Member], +) -> Result, Pass2RuleError> { + let columns: BTreeSet<&SourceColumn> = cluster + .iter() + .flat_map(|member| member.selection.keys()) + .collect(); + let mut hull = BTreeMap::new(); + let mut keys: BTreeSet = cluster + .iter() + .flat_map(|member| member.keys.iter().cloned()) + .collect(); + for column in columns { + let constraints: Option> = cluster + .iter() + .map(|member| member.selection.get(column)) + .collect(); + let union = constraints.and_then(|constraints| { + if constraints + .iter() + .all(|constraint| *constraint == constraints[0]) + { + return Some(constraints[0].clone()); + } + let mut values: Vec = Vec::new(); + for constraint in constraints { + let Constraint::In(more) = constraint else { + return None; + }; + values.extend( + more.iter() + .filter(|value| !values.contains(value)) + .cloned() + .collect::>(), + ); + } + Some(Constraint::In(values)) + }); + if let Some(union) = union { + hull.insert(column.clone(), union); + } + if cluster + .iter() + .any(|member| member.selection.get(column) != hull.get(column)) + { + keys.insert(column.clone()); + } + } + if let Some(member) = cluster.iter().find(|member| { + member.selection == hull && member.keys.iter().cloned().collect::>() == keys + }) { + return Ok(Some(member.group)); + } + let Some(base) = cluster.iter().find(|member| member.selection == hull) else { + return Ok(None); + }; + let rows_lineage = &memo.groups()[base.rows.0 as usize] + .group_properties + .column_lineage; + let mut key_columns = Vec::new(); + for key in &keys { + let Some(column) = column_of(rows_lineage, key) else { + return Ok(None); + }; + key_columns.push(column); + } + key_columns.sort_unstable(); + let mut expression = base.expression.clone(); + match &mut expression { + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) + | Operator::NonASAP(NonASAPOp::Aggregate { reduction, .. }) => { + *reduction = Reduction::by(key_columns) + } + _ => unreachable!("a member is a SummaryAgg or an Aggregate"), + } + let finest = memo.find_or_add_group(expression.clone())?; + for theta in listed_build_parameters(&expression) { + add_candidate_if_output_matches( + memo, + finest, + expression.clone(), + theta, + OriginOfCandidateForMemoGroup::Rollup, + )?; + } + Ok(Some(finest)) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup/member.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/member.rs new file mode 100644 index 000000000..8e9fa6669 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/member.rs @@ -0,0 +1,270 @@ +//! One aggregate group as the rollup rule reads it ([`Member`]): its keys, +//! selection and definition, and whether two are of one cluster. + +use std::collections::BTreeMap; + +use asap_types::ir::operator::arguments::SourceColumn; +use asap_types::ir::operator::Reduction; +use asap_types::ir::properties::selection::{constraint_of, Constraint}; +use asap_types::ir::schema::{DataType, FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{AggregateDefinition, ColumnLineage, Memo, MemoGroupId}; +use asap_types::summary_type::SummaryTypeCapability; + +use super::rewrite::rolled_up; + +/// One aggregate group as a rollup rule reads it. +pub(super) struct Member { + pub(super) group: MemoGroupId, + /// The group's expression. + pub(super) expression: Operator, + /// The rows it aggregates: the `SummaryAgg`'s or `Aggregate`'s input. + pub(super) rows: MemoGroupId, + /// Its group-by keys, in output order, by the source column they copy. + pub(super) keys: Vec, + /// Its coverage's selection, one box. + pub(super) selection: BTreeMap, + /// Its coverage's definition without its keys: the top operator's + /// reduction (and an `Aggregate`'s output names) blanked, and the input + /// written out down to the source ([`Member::see_through_per_series_layers`]), + /// so equal computations over one source compare equal whatever their + /// keys, label selections and the labels their scans list. + pub(super) definition: AggregateDefinition, +} + +impl Member { + pub(super) fn of(memo: &Memo, group: MemoGroupId) -> Option { + let memo_group = &memo.groups()[group.0 as usize]; + let (rows, by) = match &memo_group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + reduction: Reduction::Reduce(by), + .. + }) if SummaryTypeCapability::merges_groups(summary_type) => (*child, by), + Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) if filters.iter().all(Option::is_none) + && measures + .iter() + .all(|measure| rolled_up(measure, 0).is_some()) => + { + (*child, by) + } + _ => return None, + }; + if by.is_without() { + return None; + } + let coverage = memo_group.group_properties.aggregate_coverage.as_ref()?; + let [selection] = coverage.selection() else { + return None; + }; + let rows_lineage = &memo.groups()[rows.0 as usize] + .group_properties + .column_lineage; + let keys = by + .keys() + .iter() + .map(|&key| match rows_lineage.get(key)? { + ColumnLineage::SourceColumn(column) => Some(column.clone()), + ColumnLineage::Computed { .. } => None, + }) + .collect::>>()?; + let mut definition = coverage.definition().clone(); + match definition.unlifted_operators.first_mut()? { + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => { + *reduction = Reduction::PerEntity + } + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + output_names, + .. + }) => { + *reduction = Reduction::PerEntity; + output_names.clear(); + } + _ => return None, + } + let mut selection = selection.columns().clone(); + Self::see_through_per_series_layers(memo, &mut definition, &mut selection); + Some(Self { + group, + expression: memo_group.expression.clone(), + rows, + keys, + selection, + definition, + }) + } + + /// Writes the input of `definition` out down to its source. Each + /// per-series layer below it ([`per_series_layer`]: a `TimeRange`, a + /// `TimeShift`, a per-series `Aggregate`) becomes an unlifted operator, + /// so two members compare equal only over the same range and the same + /// per-series function. A time-series `Scan` under them is the last + /// unlifted operator: each label matcher on it moves into `selection` + /// (by source column), since a label is constant within a series and a + /// per-series layer keeps or drops whole series alike; the rest stay + /// its predicates. Its schema is dropped when nothing left names a + /// column by position ([`names_columns_by_position`]): the labels a + /// scan lists are those its query reads, its rows (one per sample of + /// each series, the series identity holding every label) are the same + /// whatever it lists, and members name their keys and selections by + /// source column. + fn see_through_per_series_layers( + memo: &Memo, + definition: &mut AggregateDefinition, + selection: &mut BTreeMap, + ) { + while let Some(input) = definition.input { + let expression = &memo.groups()[input.0 as usize].expression; + if !per_series_layer(expression) { + break; + } + definition + .unlifted_operators + .push(expression.map_children(|_| None)); + definition.input = Some(*expression.children()[0]); + } + // The bottom `Scan`: the input group, or the last unlifted operator + // when the coverage lifted one of its predicates already. + let (scan, lineage) = match definition.input { + Some(input) => { + let scan_group = &memo.groups()[input.0 as usize]; + ( + scan_group.expression.map_children(|&child| Some(child)), + Some(&scan_group.group_properties.column_lineage), + ) + } + None => match definition.unlifted_operators.last() { + Some(last) => (last.clone(), None), + None => return, + }, + }; + let Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) = scan + else { + return; + }; + let mut kept = Vec::new(); + for predicate in predicates { + let label_matcher = constraint_of(&predicate.0).and_then(|(column, constraint)| { + let is_label = source.is_time_series() + && schema.fields.get(column)?.dtype == FieldDataType::Plain(DataType::Utf8); + match lineage?.get(column)? { + ColumnLineage::SourceColumn(source_column) + if is_label && !selection.contains_key(source_column) => + { + Some((source_column.clone(), constraint)) + } + _ => None, + } + }); + match label_matcher { + Some((column, constraint)) => { + selection.insert(column, constraint); + } + None => kept.push(predicate), + } + } + if definition.input.take().is_none() { + definition.unlifted_operators.pop(); + } + let schema = if source.is_time_series() + && kept.is_empty() + && !definition + .unlifted_operators + .iter() + .any(names_columns_by_position) + { + Schema::default() + } else { + schema + }; + definition + .unlifted_operators + .push(Operator::NonASAP(NonASAPOp::Scan { + source, + predicates: kept, + schema, + })); + } + + /// Whether `other` computes the same thing over the same time window: + /// equal definitions without keys and equal interval constraints. + pub(super) fn same_cluster(&self, other: &Self) -> bool { + self.definition == other.definition && self.intervals() == other.intervals() + } + + /// Its interval constraints (the time window, value ranges): equal + /// across a cluster, never compensated. + fn intervals(&self) -> BTreeMap<&SourceColumn, &Constraint> { + self.selection + .iter() + .filter(|(_, constraint)| matches!(constraint, Constraint::Interval { .. })) + .collect() + } +} + +/// Whether `expression` is a per-series layer a rollup sees through: a +/// `TimeRange` or `TimeShift`, or an `Aggregate` computing one value per +/// series (`Reduction::PerEntity`, such as `rate`) without row filters. +fn per_series_layer(expression: &Operator) -> bool { + match expression { + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => true, + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + filters, + having: None, + .. + }) => filters.iter().all(Option::is_none), + _ => false, + } +} + +/// Whether unlifted operator `operator` names a column of its input by +/// position, so it means the same over two scans only when they list the +/// same columns: a measure over a column (`SUM(col)`; a PromQL measure +/// reads the implicit sample value and names none), a row filter, or any +/// operator other than a `SummaryAgg`, an `Aggregate`, a `TimeRange` and a +/// `TimeShift`. The top operator's keys are blanked, and a `SummaryAgg`'s +/// update names its columns by name. +fn names_columns_by_position(operator: &Operator>) -> bool { + match operator { + Operator::ASAP(ASAPOp::SummaryAgg { filter, .. }) => filter.is_some(), + Operator::NonASAP(NonASAPOp::Aggregate { + measures, + filters, + having, + .. + }) => { + having.is_some() + || filters.iter().any(Option::is_some) + || measures + .iter() + .any(|measure| !measure.input_cols().is_empty()) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => false, + _ => true, + } +} + +/// The column of `lineage` that copies `column`, when exactly one does. +pub(super) fn column_of(lineage: &[ColumnLineage], column: &SourceColumn) -> Option { + let mut at = lineage + .iter() + .enumerate() + .filter(|(_, known)| matches!(known, ColumnLineage::SourceColumn(c) if c == column)) + .map(|(index, _)| index); + let first = at.next()?; + at.next().is_none().then_some(first) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup/mod.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/mod.rs new file mode 100644 index 000000000..fb3c77bae --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/mod.rs @@ -0,0 +1,57 @@ +//! Step 1, rollup from the finest common group (concepts doc, section 3): +//! a coarse aggregate group $G_c$ computed from a finer group $G_f$ of the +//! same definition, and per cluster of equal definitions one finest common +//! group $G^*$ every member can roll up from. +//! +//! - `member`: one aggregate group as the rule reads it, and its cluster. +//! - `finest_common_group`: a cluster's finest common group $G^*$. +//! - `rewrite`: the candidate computing a coarse group from a finer one. + +mod finest_common_group; +mod member; +mod rewrite; + +use crate::Pass2RuleError; +use asap_types::stages::{Memo, MemoGroupId}; +use finest_common_group::finest_common_group; +use member::Member; +use rewrite::rollup_from; + +/// Adds, to each aggregate group $G_c$, a candidate computing it from each +/// finer group $G_f$ of its cluster ([`rollup_from`]), after synthesizing +/// each cluster's finest common group ([`finest_common_group`]). A cluster +/// is the summary state groups (`SummaryAgg` of a summary type whose +/// states merge across groups) or exact `Aggregate` groups (of sums, +/// counts, minima and maxima) with equal definitions apart from their keys +/// ([`Member::definition`]) and equal interval constraints (the time +/// window). +pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let mut clusters: Vec> = Vec::new(); + for index in 0..memo.groups().len() { + let Some(member) = Member::of(memo, MemoGroupId(index as u32)) else { + continue; + }; + match clusters + .iter_mut() + .find(|cluster| cluster[0].same_cluster(&member)) + { + Some(cluster) => cluster.push(member), + None => clusters.push(vec![member]), + } + } + for mut cluster in clusters.into_iter().filter(|cluster| cluster.len() > 1) { + if let Some(finest) = finest_common_group(memo, &cluster)? { + if cluster.iter().all(|member| member.group != finest) { + cluster.extend(Member::of(memo, finest)); + } + } + for coarse in &cluster { + for fine in &cluster { + if coarse.group != fine.group { + rollup_from(memo, coarse, fine)?; + } + } + } + } + Ok(()) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup/rewrite.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/rewrite.rs new file mode 100644 index 000000000..7f10105dc --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/rollup/rewrite.rs @@ -0,0 +1,220 @@ +//! The candidate computing a coarse group from a finer one of its cluster: +//! a regrouping merge or rollup aggregate over the finer group, under a +//! compensating `Filter` when its selection is wider. + +use crate::rules::candidates::{add_candidates_per_input_parameters, add_group_computing_itself}; +use crate::Pass2RuleError; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::properties::selection::Constraint; +use asap_types::ir::scalar::{CompareOpKind, ExprSemantics, Predicate, ScalarExpr, ScalarValue}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; + +use super::member::{column_of, Member}; + +/// Adds to `coarse` the candidate computing it from `fine` when (concepts +/// doc, section 3, conditions 1–5) its keys are among `fine`'s, its +/// selection lies inside `fine`'s, narrower only on `fine`'s keys, and the +/// cluster's equal definition and time window hold: +/// `Rollup(by coarse's keys)(Filter(compensation, fine))`, without the +/// `Filter` when the selections are equal. States roll up with a +/// regrouping `SummaryMerge`; sums and counts with a sum, minima and maxima +/// with themselves, each under the coarse column's name. The memo checks +/// the result is the coarse group's: a SQL global count (0 over no row) +/// does not roll up from a grouped one (its sum is NULL over no group), +/// while a grouped count does (every group it returns counts a row). +pub(super) fn rollup_from( + memo: &mut Memo, + coarse: &Member, + fine: &Member, +) -> Result<(), Pass2RuleError> { + if !coarse.keys.iter().all(|key| fine.keys.contains(key)) { + return Ok(()); + } + let mut compensation = Vec::new(); + for column in coarse.selection.keys().chain(fine.selection.keys()) { + match (coarse.selection.get(column), fine.selection.get(column)) { + (Some(narrow), wide) if Some(narrow) != wide => { + let within = match (narrow, wide) { + (_, None) => true, + (Constraint::In(values), Some(Constraint::In(wider))) => { + values.iter().all(|value| wider.contains(value)) + } + _ => false, + }; + if !within || !fine.keys.contains(column) { + return Ok(()); + } + if !compensation.iter().any(|(known, _)| known == column) { + compensation.push((column.clone(), narrow.clone())); + } + } + (None, Some(_)) => return Ok(()), + _ => {} + } + } + if compensation.is_empty() && coarse.keys.len() == fine.keys.len() { + return Ok(()); + } + let lineage = &memo.groups()[fine.group.0 as usize] + .group_properties + .column_lineage; + let mut conjuncts = Vec::new(); + for (column, constraint) in &compensation { + let Some(at) = column_of(lineage, column) else { + return Ok(()); + }; + // The comparison of the frontend that reads the column's source. + let semantics = if column.source.is_time_series() { + ExprSemantics::Promql + } else { + ExprSemantics::Sql + }; + let Some(conjunct) = selecting(at, constraint, semantics) else { + return Ok(()); + }; + conjuncts.push(conjunct); + } + let Some(group_by) = coarse + .keys + .iter() + .map(|key| column_of(lineage, key)) + .collect::>>() + else { + return Ok(()); + }; + // The compensating `Filter` stays pending, named by the id it would get + // as a new group, until the rollup over it derives the coarse group's + // schema (`Memo::output_schema_of_pending_expression`): a rollup the + // memo would reject adds no group. + let filter = (!conjuncts.is_empty()).then(|| { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(if conjuncts.len() == 1 { + conjuncts.remove(0) + } else { + ScalarExpr::BoolAnd(conjuncts) + }), + child: fine.group, + }) + }); + let mut pending = Vec::new(); + let input = match &filter { + None => fine.group, + Some(filter) => { + let schema = memo.output_schema_of_pending_expression(&[], filter)?; + pending.push((filter.clone(), schema)); + MemoGroupId(u32::try_from(memo.groups().len()).expect("at most u32::MAX groups")) + } + }; + let rollup = match &coarse.expression { + Operator::ASAP(ASAPOp::SummaryAgg { .. }) => Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![input], + group_by: Some(group_by), + }), + Operator::NonASAP(NonASAPOp::Aggregate { + measures, filters, .. + }) => { + let Some(measures) = measures + .iter() + .enumerate() + .map(|(index, measure)| rolled_up(measure, fine.keys.len() + index)) + .collect::>>() + else { + return Ok(()); + }; + // Each rolled-up measure keeps the coarse column's name: a sum + // of counts is named `count`, as the count it computes. + let output_names = memo.groups()[coarse.group.0 as usize] + .group_properties + .schema + .fields + .iter() + .skip(coarse.keys.len()) + .map(|field| field.name.clone()) + .collect(); + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(group_by), + measures, + output_names, + filters: vec![None; filters.len()], + having: None, + child: input, + }) + } + _ => unreachable!("a member is a SummaryAgg or an Aggregate"), + }; + if memo + .output_schema_of_pending_expression(&pending, &rollup) + .ok() + .as_ref() + != Some( + &memo.groups()[coarse.group.0 as usize] + .group_properties + .schema, + ) + { + return Ok(()); + } + let rollup = match filter { + None => rollup, + Some(filter) => { + let filter = + add_group_computing_itself(memo, filter, OriginOfCandidateForMemoGroup::Rollup)?; + rollup.map_children(|_| filter) + } + }; + add_candidates_per_input_parameters( + memo, + coarse.group, + rollup, + OriginOfCandidateForMemoGroup::Rollup, + ) +} + +/// The measure rolling `measure`'s finalized values up from column +/// `column` of the finer result: a sum or count by a sum, a minimum or +/// maximum by itself; `None` for any other measure (it rolls up only from +/// states). +pub(super) fn rolled_up(measure: &AggIntent, column: usize) -> Option { + let col = Some(column); + Some(match measure { + AggIntent::Sum { .. } | AggIntent::Count => AggIntent::Sum { col }, + AggIntent::Min { .. } => AggIntent::Min { col }, + AggIntent::Max { .. } => AggIntent::Max { col }, + _ => return None, + }) +} + +/// The predicate selecting the rows whose column `at` meets `constraint`, +/// with the comparison `semantics` of the frontend reading the column's +/// source: `at = v`, `at != v`, or their disjunction (`IN`) or conjunction +/// (`NOT IN`) over several values. A PromQL `!=` keeps a series without the +/// label (its value is the empty string), which only the frontend's +/// comparison says. `None` for an interval (never a compensation: the time +/// window is equal across a cluster). +fn selecting( + at: usize, + constraint: &Constraint, + semantics: ExprSemantics, +) -> Option> { + let compare = |op: CompareOpKind, value: &ScalarValue| ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(at)), + op, + right: Box::new(ScalarExpr::Literal(value.clone())), + semantics, + }; + let (op, values, combine): (_, _, fn(Vec<_>) -> _) = match constraint { + Constraint::In(values) => (CompareOpKind::Eq, values, ScalarExpr::BoolOr), + Constraint::NotIn(values) => (CompareOpKind::Ne, values, ScalarExpr::BoolAnd), + Constraint::Interval { .. } => return None, + }; + Some(match &values[..] { + [value] => compare(op, value), + values => combine( + values + .iter() + .map(|value| compare(op.clone(), value)) + .collect(), + ), + }) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs b/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs index f90858eff..d13d50d35 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs @@ -1,14 +1,88 @@ //! Step 2, top-k reuse: an exact top-k reads a larger top-k of the same //! input and partition, whose ranked rows hold its own. -use crate::rules::Pass2RuleError; -use asap_types::stages::Memo; +use crate::rules::candidates::add_candidate_if_output_matches; +use crate::Pass2RuleError; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; -/// To each top-k group another top-k group over the same input group and -/// partition with the largest `k` serves, the candidate `TopK(k)` over that -/// group's ranked rows: the `k` highest of the `k_max` highest rows are the -/// `k` highest. Only exact ranked rows give the reuse its answer. +/// To each top-k group (`Aggregate` of one `TopK` measure, partitioned `by` +/// keys, without filters or `HAVING`) another top-k group over the same +/// input group and partition with the largest `k` serves, the candidate +/// `TopK(k)` over that group's ranked rows, partitioned by its leading key +/// columns: the `k` highest of the `k_max` highest rows are the `k` +/// highest. Only exact ranked rows give the reuse its answer: a ranking has +/// no error contract (`asap_accuracy::ErrorContract`), so the edge +/// from an approximate producer (a heap's readout) fails the metric rule +/// when a query with an accuracy target reaches it. pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: step 2, top-k reuse") + let topks: Vec<(MemoGroupId, MemoGroupId, Reduction, usize)> = (0..memo.groups().len()) + .filter_map(|index| { + let group = MemoGroupId(index as u32); + let (child, reduction, k) = topk(&memo.groups()[index].expression)?; + Some((group, child, reduction, k)) + }) + .collect(); + for (group, child, reduction, k) in &topks { + let Some((largest, k_max)) = topks + .iter() + .filter(|(_, other_child, other_reduction, _)| { + other_child == child && other_reduction == reduction + }) + .map(|(other, _, _, other_k)| (*other, *other_k)) + .max_by_key(|&(other, other_k)| (other_k, std::cmp::Reverse(other))) + else { + continue; + }; + if k_max <= *k { + continue; + } + let Operator::NonASAP(NonASAPOp::Aggregate { + measures, + output_names, + filters, + .. + }) = memo.groups()[group.0 as usize].expression.clone() + else { + continue; + }; + let keys = reduction.group_keys().map_or(0, |by| by.keys().len()); + add_candidate_if_output_matches( + memo, + *group, + Operator::NonASAP(NonASAPOp::Aggregate { + child: largest, + reduction: Reduction::by((0..keys).collect()), + measures, + output_names, + filters, + having: None, + }), + None, + OriginOfCandidateForMemoGroup::TopKReuse, + )?; + } + Ok(()) +} + +/// The input group, partition and `k` of a top-k group: one `TopK` measure, +/// `by` keys (not `without`), no measure filter, no `HAVING`. +fn topk(operator: &Operator) -> Option<(MemoGroupId, Reduction, usize)> { + let Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: reduction @ Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) = operator + else { + return None; + }; + let [AggIntent::TopK { k, .. }] = measures.as_slice() else { + return None; + }; + (!by.is_without() && filters.iter().all(Option::is_none)) + .then(|| (*child, reduction.clone(), *k)) } diff --git a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs new file mode 100644 index 000000000..5213cae39 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs @@ -0,0 +1,149 @@ +//! Fixtures this crate's own tests share, beside the memo builders of +//! `asap_types::test_support::memo_builders` (groups with the candidates +//! Pass 1 would add, queries), so no test calls another stage crate: the +//! group builders Pass 2's tests alone use, Pass 2's run and readers of +//! its output. +use std::collections::BTreeSet; + +use asap_accuracy::NoMeasuredGuarantees; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PrunedLogicalCandidate, + RequiredAccuracy, +}; +use asap_types::summary_type::ExactKind; +use asap_types::test_support::memo_builders::{candidates_of, group, groups_where}; +use asap_types::test_support::memo_fixtures; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RepeatedDemand, + RepetitionInterval, TimestampMs, +}; + +use crate::generate_logical_asap_memo_candidates; + +/// An exact accumulator state of `kind` over `window` grouped by +/// `reduction`, read by a `FinalizeExactAccumulator` group (the query +/// root): what Pass 1 offers for `sum_over_time` (per series) or for a +/// `sum by (keys)` of `count_over_time` (the count by keys). +pub fn exact_state( + memo: &mut Memo, + window: MemoGroupId, + kind: ExactKind, + reduction: Reduction, +) -> (MemoGroupId, MemoGroupId) { + let state = group( + memo, + memo_fixtures::state_grouped_by(FieldDataType::exact(kind), reduction, window), + ); + let finalized = group( + memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + ); + (state, finalized) +} + +/// A query repeating every `interval_ms` at phase 0 (`FixedIntervalAt`). +pub fn entry(interval_ms: u32) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("q".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, + } +} + +pub fn pass2(memo: Memo, entries: &[QueryWorkloadEntry]) -> LogicalASAPMemoCandidates { + generate_logical_asap_memo_candidates(memo, entries, &NoMeasuredGuarantees).unwrap() +} + +/// The edges into input `input` of `consumer`: each producer and α^log. +pub fn edges_into( + logical: &LogicalASAPMemoCandidates, + consumer: CandidateForMemoGroupId, + input: usize, +) -> Vec<(CandidateForMemoGroupId, bool)> { + logical + .edges + .iter() + .filter(|edge| { + edge.consumer_candidate_id == consumer && edge.input_index_of_consumer == input + }) + .map(|edge| { + ( + edge.producer_candidate_id, + edge.satisfies_logical_requirement, + ) + }) + .collect() +} + +/// The pruned entry of `candidate`, if Pass 2 eliminated it. +pub fn pruned_entry( + logical: &LogicalASAPMemoCandidates, + candidate: CandidateForMemoGroupId, +) -> Option<&PrunedLogicalCandidate> { + logical + .pruned + .iter() + .find(|pruned| pruned.candidate == candidate) +} + +/// The required accuracy of each readout candidate of `group` (its only +/// input's). +pub fn requirements_of(memo: &Memo, group: MemoGroupId) -> Vec> { + candidates_of(memo, group) + .iter() + .filter(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + ) + }) + .map(|c| c.input_requirements[0].required_accuracy) + .collect() +} + +/// The tumbling-window widths of the chains whose merges state group +/// `state` has as candidates. +pub fn chain_widths_serving(memo: &Memo, state: MemoGroupId) -> BTreeSet { + candidates_of(memo, state) + .iter() + .filter_map(|c| match &c.expression.operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + }) => chain_width_below(memo, children[0]), + _ => None, + }) + .collect() +} + +/// The width of the tumbling-window chain `group` reads through +/// `TimeRange` and `TimeShift` groups, if any. +pub fn chain_width_below(memo: &Memo, mut group: MemoGroupId) -> Option { + loop { + match &memo.groups()[group.0 as usize].expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => return Some(*tumbling_window_width_ms), + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => group = *child, + _ => return None, + } + } +} + +/// The group whose expression is `operator`, if the memo has one. +pub fn group_of(memo: &Memo, operator: &Operator) -> Option { + groups_where(memo, |expression| expression == operator) + .first() + .copied() +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests.rs b/crates/stage1-pass2-reuse-candidates/src/tests.rs new file mode 100644 index 000000000..68ccba04c --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests.rs @@ -0,0 +1,11 @@ +//! Pass 2's tests, one module per step. Each reads what the rules, the +//! readout requirements, dead-candidate elimination and the edges give for +//! a memo built from `crate::test_fixtures`. +mod chains; +mod dead_candidates; +mod edges; +mod evaluation_grids; +mod exact_only_groups; +mod readout_requirements; +mod rollup; +mod topk; diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/chains.rs b/crates/stage1-pass2-reuse-candidates/src/tests/chains.rs new file mode 100644 index 000000000..f588f21ed --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/chains.rs @@ -0,0 +1,343 @@ +//! Step 5: chain groups. +use std::collections::BTreeSet; +use std::time::Duration; + +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, TimeShift}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, RootPruneReason, +}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{ExactKind, SketchAlgorithm}; +use asap_types::workload::{ + DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, +}; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::{range, shift}; + +/// A mergeable state over a range gets one chain group per width with a +/// build per surviving θ; its readout reads the builds and the merges, +/// one per θ. +#[test] +fn a_mergeable_state_reads_a_chain_with_one_build_per_surviving_parameters() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(120_000)]); + let memo = &logical.memo; + assert_eq!(chain_widths_serving(memo, state), BTreeSet::from([60_000])); + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) + }); + let listed = parameter_list(SketchAlgorithm::Kll).len(); + assert_eq!(candidates_of(memo, chains[0]).len(), listed); + // The builds and one merge per θ. + assert_eq!( + edges_into(&logical, own_candidate(readout), 0).len(), + 2 * listed + ); +} + +/// No chain for a summary type whose states do not merge (an instant +/// rate), under `@ end()`, or for a query without a known grid. +#[test] +fn no_chain_without_a_merge_an_anchor_or_a_grid() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let (irate, irate_root) = + exact_state(&mut memo, window, ExactKind::IRate, Reduction::PerEntity); + let anchored = group(&mut memo, shift(0, Some(AtModifier::End), input)); + let anchored_window = group(&mut memo, range(300_000, anchored)); + let (pinned, pinned_root) = exact_state( + &mut memo, + anchored_window, + ExactKind::Sum, + Reduction::PerEntity, + ); + let other = scan_group(&mut memo, "n", &["job"], vec![]); + let other_window = group(&mut memo, range(300_000, other)); + let (no_grid, no_grid_root) = exact_state( + &mut memo, + other_window, + ExactKind::Sum, + Reduction::PerEntity, + ); + for root in [irate_root, pinned_root, no_grid_root] { + query(&mut memo, root, None); + } + let without_phase = QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + ..entry(60_000) + }; + let logical = pass2(memo, &[entry(60_000), entry(60_000), without_phase]); + for state in [irate, pinned, no_grid] { + assert!( + chain_widths_serving(&logical.memo, state).is_empty(), + "G{}", + state.0 + ); + } + assert!(groups_where(&logical.memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + )) + .is_empty()); +} + +/// A chain serves a reader only when its width divides the reader's +/// evaluation grid: `quantile_over_time(0.9, m[30s])` read every 15 s and +/// every 10 s is one state group evaluated on the 5 s grid, so of its 5, +/// 10 and 15 s chains only the 5 s one has edges, and the merges over the +/// others are pruned (a 15 s chain would answer the 10 s reader at times +/// it does not end on). +#[test] +fn a_chain_whose_width_does_not_divide_the_readers_grid_fails_its_edge() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 30); + query(&mut memo, readout, None); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(15_000), entry(10_000)]); + let memo = &logical.memo; + assert_eq!( + chain_widths_serving(memo, state), + BTreeSet::from([5_000, 10_000, 15_000]) + ); + for edge in &logical.edges { + if let Some(width) = chain_width_below(memo, edge.producer_candidate_id.group) { + assert_eq!(width, 5_000, "{edge:?}"); + } + } + let mut live_widths = BTreeSet::new(); + for (index_in_group, merge) in candidates_of(memo, state).iter().enumerate() { + let Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) = &merge.expression.operator + else { + continue; + }; + let merge = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + match pruned_entry(&logical, merge) { + None => live_widths.extend(chain_width_below(memo, children[0])), + Some(pruned) => assert!( + pruned.causes.iter().any(|cause| matches!( + cause.root, + RootPruneReason::ChainWidthDoesNotDivideEvaluationGrid { + evaluation_period_ms: Some(5_000), + .. + } + )), + "{pruned:?}" + ), + } + } + assert_eq!(live_widths, BTreeSet::from([5_000])); +} + +/// A state under `@ T` adds no width; it reads a chain another query's +/// width created when that width divides T. +#[test] +fn an_absolute_anchor_reuses_only_existing_chains() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + let mut pinned = Vec::new(); + // T = …040 s lies on the 60 s grid; T = …010 s does not. + for at_ms in [1_700_000_040_000, 1_700_000_010_000] { + let at = group( + &mut memo, + shift(0, Some(AtModifier::Timestamp(at_ms)), input), + ); + let at_window = group(&mut memo, range(300_000, at)); + let (state, root) = exact_state(&mut memo, at_window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + pinned.push(state); + } + let logical = pass2(memo, &vec![entry(60_000); 3]); + let widths = |state| chain_widths_serving(&logical.memo, state); + assert_eq!(widths(state), BTreeSet::from([60_000])); + assert_eq!(widths(pinned[0]), BTreeSet::from([60_000])); + assert!(widths(pinned[1]).is_empty()); +} + +/// A state under `@ T` reads the chains the other states create whatever +/// the memo order: listed before the query whose 60 s grid creates the +/// chain, the state pinned at T = …040 s still reads it. +#[test] +fn an_absolute_anchor_reuses_a_chain_a_later_group_creates() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let at = group( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_700_000_040_000)), input), + ); + let at_window = group(&mut memo, range(300_000, at)); + let (pinned, pinned_root) = + exact_state(&mut memo, at_window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, pinned_root, None); + let window = group(&mut memo, range(300_000, input)); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + assert!(pinned.0 < state.0, "the pinned state comes first"); + let logical = pass2(memo, &vec![entry(60_000); 2]); + let widths = |state| chain_widths_serving(&logical.memo, state); + assert_eq!(widths(state), BTreeSet::from([60_000])); + assert_eq!(widths(pinned), BTreeSet::from([60_000])); +} + +/// Chain groups come after the rollup +/// rule and the reach along candidate edges, so a state's widths come from +/// every query that reaches it through a candidate. +/// +/// A `sum by (job, instance) (count_over_time(m[5m]))` every 60 s; B +/// `sum by (job) (count_over_time(m[5m]))` every 90 s (both at phase 0). +/// B's state rolls up A's, so both reach A's state, whose widths are the +/// gcd closure of {60 s, 90 s} with 300 s: 30 s and 60 s. +#[test] +fn chain_groups_run_after_reach_so_a_rolled_up_query_adds_its_grid() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job", "instance"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let (by_job_instance, a) = exact_state( + &mut memo, + window, + ExactKind::Count, + Reduction::by(vec![2, 3]), + ); + let (by_job, b) = exact_state(&mut memo, window, ExactKind::Count, Reduction::by(vec![2])); + let query_a = query(&mut memo, a, Some(AccuracyTarget::exact())); + let query_b = query(&mut memo, b, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(90_000)]); + let memo = &logical.memo; + assert!( + candidates_of(memo, by_job).iter().any(|c| { + c.origin == OriginOfCandidateForMemoGroup::Rollup + && c.expression.operator.children() == vec![&by_job_instance] + }), + "B's state reads A's through a rollup" + ); + let reached_by = memo.groups()[by_job_instance.0 as usize].reached_by_queries(); + assert!( + reached_by.contains(&query_a) && reached_by.contains(&query_b), + "A's state is reached by A and B: {reached_by:?}" + ); + assert_eq!( + chain_widths_serving(memo, by_job_instance), + BTreeSet::from([30_000, 60_000]) + ); + assert_eq!(chain_widths_serving(memo, by_job), BTreeSet::from([30_000])); +} + +/// A window read at an `offset` takes the +/// offset into the gcd, since its windows' edges move by it. +/// `sum_over_time(m[5m] offset 45s)` every 60 s (phase 0): widths divide +/// 60 s, 300 s and 45 s, so 15 s only. +#[test] +fn an_offset_enters_the_gcd_of_the_widths() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let shifted = group(&mut memo, shift(45_000, None, input)); + let window = group(&mut memo, range(300_000, shifted)); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + chain_widths_serving(&logical.memo, state), + BTreeSet::from([15_000]) + ); +} + +/// A tumbling-window chain is a group +/// of its own, `SummaryTumblingWindowChain` over the rows with a width and +/// no range, offset or window count, so every range and offset of one rows +/// and width reads the same chain group through `TimeRange` (and +/// `TimeShift`) groups and a `SummaryMerge`. +/// +/// A `sum_over_time(m[5m])`, B `sum_over_time(m[5m] offset 1h)`, C +/// `sum_over_time(m[1h])`, each every 60 s (phase 0): one chain group Gw of +/// width 60 s over the scan of m; the groups `TimeShift(1h, Gw)`, +/// `TimeRange(5m, Gw)`, `TimeRange(5m, TimeShift(1h, Gw))` and +/// `TimeRange(1h, Gw)`; each state group has a `SummaryMerge` candidate +/// over its `TimeRange` group. +#[test] +fn one_range_free_chain_group_serves_every_range_and_offset() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let five_minutes = group(&mut memo, range(300_000, input)); + let shifted_input = group(&mut memo, shift(3_600_000, None, input)); + let five_minutes_an_hour_ago = group(&mut memo, range(300_000, shifted_input)); + let one_hour = group(&mut memo, range(3_600_000, input)); + let mut states = Vec::new(); + for window in [five_minutes, five_minutes_an_hour_ago, one_hour] { + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, Some(AccuracyTarget::exact())); + states.push(state); + } + let logical = pass2(memo, &vec![entry(60_000); 3]); + let memo = &logical.memo; + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) + }); + let [chain] = chains[..] else { + panic!("one chain group, found {chains:?}"); + }; + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + tumbling_window_width_ms, + .. + }) = &memo.groups()[chain.0 as usize].expression + else { + unreachable!(); + }; + assert_eq!(*tumbling_window_width_ms, 60_000); + assert_eq!( + *child, input, + "the chain reads the scan: no range or shift below it" + ); + let range_over = |range_s: u64, child: MemoGroupId| { + group_of( + memo, + &Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(range_s), + kind: TimeRangeKind::Range, + child, + }), + ) + }; + let shifted_chain = group_of( + memo, + &Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: chain, + }), + ) + .expect("TimeShift(1h, Gw)"); + let ranges = [ + range_over(300, chain).expect("TimeRange(5m, Gw)"), + range_over(300, shifted_chain).expect("TimeRange(5m, TimeShift(1h, Gw))"), + range_over(3600, chain).expect("TimeRange(1h, Gw)"), + ]; + for (state, range) in states.iter().zip(ranges) { + assert!( + candidates_of(memo, *state).iter().any(|c| matches!( + &c.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: None }) + if *children == vec![range] + )), + "G{} reads G{} through a SummaryMerge", + state.0, + range.0 + ); + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs new file mode 100644 index 000000000..158e83b7c --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs @@ -0,0 +1,118 @@ +//! Steps 4 and 5: dead-candidate elimination and Pass 2's pruned list. +use std::collections::BTreeSet; + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{SketchAlgorithm, SketchParams}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::{arithmetic, range}; + +/// The concepts doc example "ratio of quantiles": +/// `quantile_over_time(0.99, m[5m]) / quantile_over_time(0.5, m[5m])`, +/// relative error 0.05, every 60 s; Pass 1 offers KLL and DDSketch to both +/// quantiles. The ratio carries only relative error, so each KLL readout is +/// pruned by the metric rule and every KLL build through it. The readouts +/// of DDSketch get the levels 0.02 … 0.001 (cap 0.045), so the builds with +/// alpha 0.1 and 0.05 miss every level they could serve. Chain groups are +/// added only for the surviving builds. +#[test] +fn dead_candidates_are_pruned_and_listed_before_chain_groups() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &algorithms); + let p50 = quantile_over_time_readouts(&mut memo, window, 0.5, &algorithms); + let ratio = group( + &mut memo, + arithmetic(ArithmeticOpKind::Div, p99.result, p50.result), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, ratio, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + let memo = &logical.memo; + let (kll_state, ddsketch_state) = (p99.by_algorithm[0].state, p99.by_algorithm[1].state); + // Each KLL readout: the metric rule at the ratio, directly. + for quantile in [&p99, &p50] { + let kll_readout = quantile.by_algorithm[0].candidate_of_result; + let pruned = pruned_entry(&logical, kll_readout).expect("the KLL readout is pruned"); + assert!(pruned.causes.iter().all(|cause| matches!( + &cause.root, + RootPruneReason::MetricRule { producer_metric: AccuracyMetric::Rank, accepted_metrics, .. } + if accepted_metrics == &vec![AccuracyMetric::Relative] + ))); + } + // Every KLL build: the metric rule on its edge into a KLL readout, + // whose output accepts only relative error. + for index_in_group in 0..parameter_list(SketchAlgorithm::Kll).len() { + let build = CandidateForMemoGroupId { + group: kll_state, + index_in_group, + }; + let pruned = pruned_entry(&logical, build).expect("every KLL build is pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + &cause.root, + RootPruneReason::MetricRule { producer_metric: AccuracyMetric::Rank, accepted_metrics, .. } + if accepted_metrics == &vec![AccuracyMetric::Relative] + ))); + } + // DDSketch: alpha 0.1 and 0.05 pruned below every required accuracy; + // the levels of the readouts are 0.02 … 0.001. + let pruned_alphas: Vec = logical + .pruned + .iter() + .filter(|pruned| pruned.candidate.group == ddsketch_state) + .map(|pruned| { + assert!(pruned + .causes + .iter() + .all(|cause| cause.through_pruned_candidates.is_empty() + && matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. }))); + match memo + .candidate(pruned.candidate) + .logical_properties + .parameters() + { + Some(SketchParams::DDSketch { alpha }) => *alpha, + other => panic!("{other:?}"), + } + }) + .collect(); + assert_eq!(pruned_alphas, vec![0.1, 0.05]); + let readout_group = p99.by_algorithm[1].readout.group; + let mut levels: Vec = requirements_of(memo, readout_group) + .into_iter() + .map(|required| required.expect("a DDSketch readout's level").max_error) + .collect(); + levels.sort_by(f64::total_cmp); + assert_eq!(levels, vec![0.001, 0.002, 0.005, 0.01, 0.02]); + // Chain builds exist only for the five surviving alphas. + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. }) + if *summary_type == FieldDataType::Sketch(SketchAlgorithm::DDSketch)) + }); + assert_eq!(chains.len(), 1); + assert_eq!(candidates_of(memo, chains[0]).len(), 5); + assert!(groups_where(memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. }) + if *summary_type == FieldDataType::Sketch(SketchAlgorithm::Kll) + )) + .is_empty()); + // No logical edge names a pruned candidate. + let pruned: BTreeSet<_> = logical.pruned.iter().map(|p| p.candidate).collect(); + assert!(logical + .edges + .iter() + .all(|edge| !pruned.contains(&edge.producer_candidate_id) + && !pruned.contains(&edge.consumer_candidate_id))); + let _ = kll_state; +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs new file mode 100644 index 000000000..fe726caf0 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs @@ -0,0 +1,122 @@ +//! Steps 4 to 6: the logical edges Pass 2 assembles and keeps ($\alpha^{\log}$ +//! and the metric rule themselves are tested in `asap_accuracy`). + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, RequiredAccuracy, RootPruneReason, +}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::arithmetic; + +/// Without accuracy targets (accuracy not enforced) readouts get no +/// requirement and no metric rule applies: every edge is admissible. +#[test] +fn without_targets_every_edge_is_admissible() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, _, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let p50 = estimate( + &mut memo, + MemoGroupId(2), + SketchStatistic::Quantile { q: 0.5 }, + ); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + query(&mut memo, ratio, None); + let logical = pass2(memo, &[entry(60_000)]); + assert!(logical.pruned.is_empty()); + assert!(logical + .edges + .iter() + .all(|edge| edge.satisfies_logical_requirement)); +} + +/// Every candidate of an input group is an edge of each consumer input, +/// except that a candidate carrying a θ without building (a merge, a +/// `TimeRange` over a chain) reads only producers with its θ. +#[test] +fn every_candidate_of_an_input_group_is_an_edge_of_matching_parameters() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(300_000)]); + let listed = parameter_list(SketchAlgorithm::DDSketch).len(); + // The builds and one merge per θ over the 300 s chain. + assert_eq!( + edges_into(&logical, own_candidate(readout), 0).len(), + 2 * listed + ); + let memo = &logical.memo; + for (index_in_group, merge) in candidates_of(memo, state).iter().enumerate() { + if !matches!( + merge.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + ) { + continue; + } + let consumer = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + let producers = edges_into(&logical, consumer, 0); + assert_eq!(producers.len(), 1); + assert_eq!( + memo.candidate(producers[0].0) + .logical_properties + .parameters(), + merge.logical_properties.parameters() + ); + } +} + +/// A merge carries the θ of the states it reads. +/// P = `quantile_over_time(0.9, m[5m])` at rank 0.005 keeps KLL builds +/// from k = 547; Q = `quantile_over_time(0.9, m[10m])` at rank 0.01 from +/// k = 269. Both read one 60 s chain, which has the builds of both, so P's +/// state gets `SummaryMerge[k=269]` and `SummaryMerge[k=547]`: the second +/// serves P's readout, the first is pruned with `BelowRequiredAccuracy`. +#[test] +fn a_merge_is_judged_by_the_parameters_it_carries() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, p_state, p_readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let (_, _, q_readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 600); + query(&mut memo, p_readout, Some(AccuracyTarget::max_error(0.005))); + query(&mut memo, q_readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let merge_with_k = |k: u32| -> CandidateForMemoGroupId { + let index_in_group = candidates_of(memo, p_state) + .iter() + .position(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + ) && c.logical_properties.parameters() == Some(&SketchParams::Kll { k }) + }) + .unwrap_or_else(|| panic!("SummaryMerge[k={k}] in P's state group")); + CandidateForMemoGroupId { + group: p_state, + index_in_group, + } + }; + let into_p = edges_into(&logical, own_candidate(p_readout), 0); + assert!(into_p.contains(&(merge_with_k(547), true)), "{into_p:?}"); + let pruned = pruned_entry(&logical, merge_with_k(269)).expect("SummaryMerge[k=269] is pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + cause.root, + RootPruneReason::BelowRequiredAccuracy { + required: RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error, + .. + }, + .. + } if max_error == 0.005 + ))); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/evaluation_grids.rs b/crates/stage1-pass2-reuse-candidates/src/tests/evaluation_grids.rs new file mode 100644 index 000000000..07fcd3f85 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/evaluation_grids.rs @@ -0,0 +1,115 @@ +//! Steps 4 to 6: the evaluation grid of each group, per query reaching it. +use std::collections::BTreeSet; +use std::time::Duration; + +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::ExactKind; +use asap_types::workload::{ + DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, +}; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::range; + +/// A group's evaluation grid is the gcd of its reaching queries' grids; +/// `None` when one has no known grid. +#[test] +fn a_groups_evaluation_grid_is_the_gcd_of_its_queries_grids() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let (_, a) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + let other = scan_group(&mut memo, "n", &["job"], vec![]); + query(&mut memo, a, None); + query(&mut memo, a, None); + query(&mut memo, other, None); + let without_phase = QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + ..entry(60_000) + }; + let logical = pass2(memo, &[entry(60_000), entry(120_000), without_phase]); + let grid = |group: MemoGroupId| logical.evaluation_grid_of_group[group.0 as usize]; + assert_eq!(grid(window).map(|g| g.period_ms), Some(60_000)); + assert_eq!(grid(other), None); +} + +/// `max_over_time(sum_over_time(m[2m])[10m:])` read every 60 s +/// over a scan of `m`: the inner sum group and its state, the subquery +/// group and the query's root. +fn sum_over_time_in_a_subquery( + memo: &mut Memo, + resolution: Option, +) -> (MemoGroupId, MemoGroupId, MemoGroupId) { + let input = scan_group(memo, "m", &["job"], vec![]); + let window = group(memo, range(120_000, input)); + let (state, sum) = exact_state(memo, window, ExactKind::Sum, Reduction::PerEntity); + let subquery = group( + memo, + Operator::NonASAP(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(600), + resolution, + child: sum, + }), + ); + let root = group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: subquery, + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Max { col: None }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ); + query(memo, root, None); + (state, subquery, root) +} + +/// Below a subquery `[10m:30s]` the inner expression is evaluated at the +/// multiples of 30 s, whatever the outer grid: read every 60 s, the inner +/// sum's state is on the 30 s grid and reads a 30 s chain, not a 60 s one +/// whose windows end only at every other inner evaluation. +#[test] +fn below_a_subquery_the_grid_is_its_resolution() { + let mut memo = Memo::new(&DataWorkload::default()); + let (state, subquery, root) = + sum_over_time_in_a_subquery(&mut memo, Some(Duration::from_secs(30))); + let logical = pass2(memo, &[entry(60_000)]); + let period = |group: MemoGroupId| { + logical.evaluation_grid_of_group[group.0 as usize].map(|grid| grid.period_ms) + }; + assert_eq!(period(root), Some(60_000)); + assert_eq!(period(subquery), Some(60_000)); + assert_eq!(period(state), Some(30_000)); + assert_eq!( + chain_widths_serving(&logical.memo, state), + BTreeSet::from([30_000]) + ); +} + +/// A subquery without a resolution steps at the server's default +/// evaluation interval, unknown to the planner: the groups below it have +/// no known grid and get no chain. +#[test] +fn below_a_subquery_without_a_resolution_there_is_no_grid() { + let mut memo = Memo::new(&DataWorkload::default()); + let (state, subquery, _) = sum_over_time_in_a_subquery(&mut memo, None); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + logical.evaluation_grid_of_group[subquery.0 as usize].map(|grid| grid.period_ms), + Some(60_000) + ); + assert_eq!(logical.evaluation_grid_of_group[state.0 as usize], None); + assert!(groups_where(&logical.memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + )) + .is_empty()); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs new file mode 100644 index 000000000..4e9827a4a --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs @@ -0,0 +1,73 @@ +//! Step 6: the groups each query may read only exactly. +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; +use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; +use asap_types::stages::{ColumnLineage, Memo}; +use asap_types::summary_type::SketchAlgorithm; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::{arithmetic, range}; + +/// The metric rule is per query, not per group: Q0 = +/// `quantile_over_time(0.99, latency[5m]) / quantile_over_time(0.5, +/// latency[5m])` and Q1 = the same ratio `> 1`, both relative 0.05, share +/// the ratio group. Q0 may read the DDSketch readouts below it (the +/// division carries relative error); Q1's comparison has no contract, so +/// Q1 reads the ratio and every group below it only exactly. (Both read +/// the rows below a sketch build exactly: a build has no contract.) +#[test] +fn a_query_comparing_a_shared_ratio_reads_it_only_exactly() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "latency", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); + let p50 = quantile_over_time_readouts(&mut memo, window, 0.5, &[SketchAlgorithm::DDSketch]); + let ratio = group( + &mut memo, + arithmetic(ArithmeticOpKind::Div, p99.result, p50.result), + ); + let value = memo.groups()[ratio.0 as usize] + .group_properties + .column_lineage + .iter() + .position(|lineage| matches!(lineage, ColumnLineage::Computed { .. })) + .expect("the ratio computes its value"); + let above_one = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(value)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Promql, + }), + child: ratio, + }), + ); + let target = || AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + let q0 = query(&mut memo, ratio, Some(target())); + let q1 = query(&mut memo, above_one, Some(target())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let exact_only = &logical.exact_only_groups_of_query; + for group in [ratio, p99.result, p50.result, p99.by_algorithm[0].state] { + assert!(!exact_only[q0].contains(&group), "G{}", group.0); + } + for group in [ + ratio, + p99.result, + p50.result, + p99.by_algorithm[0].state, + window, + ] { + assert!(exact_only[q1].contains(&group), "G{}", group.0); + } + assert!(!exact_only[q1].contains(&above_one)); + let ddsketch_p99 = p99.by_algorithm[0].readout; + assert!( + pruned_entry(&logical, ddsketch_p99).is_none(), + "Q0 still reads the DDSketch readout" + ); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/readout_requirements.rs b/crates/stage1-pass2-reuse-candidates/src/tests/readout_requirements.rs new file mode 100644 index 000000000..625b0eb23 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/readout_requirements.rs @@ -0,0 +1,120 @@ +//! Step 3: readout requirements: what Pass 2 writes on the readouts and +//! prunes with the levels (the level math is tested in `asap_accuracy`). + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{Memo, MemoQuery, RequiredAccuracy, RootPruneReason}; +use asap_types::summary_type::{SketchAlgorithm, SketchParams}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; + +/// A single quantile query with rank error 0.01: its one readout gets that +/// requirement (λ = 1), and only the KLL builds whose rank error at δ = 1 +/// is within 0.01 satisfy it. +#[test] +fn a_single_readout_requires_the_query_target() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + query(&mut memo, readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + requirements_of(&logical.memo, readout), + vec![Some(RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error: 0.01, + failure_probability: 1.0, + })] + ); + let admitted: Vec = edges_into(&logical, own_candidate(readout), 0) + .into_iter() + .filter(|&(_, satisfies)| satisfies) + .filter_map(|(producer, _)| { + match logical + .memo + .candidate(producer) + .logical_properties + .parameters() + { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + } + }) + .collect(); + assert!(!admitted.is_empty()); + assert!(admitted.iter().all(|&k| k >= 269), "{admitted:?}"); + assert!(logical + .pruned + .iter() + .filter(|p| p.candidate.group == state) + .all(|p| p + .causes + .iter() + .all(|cause| matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. })))); +} + +/// A whole-result failure probability with the +/// number of values per evaluation unknown gives no per-value one, so the +/// readouts the query reaches are pruned with `ValuesPerEvaluationUnknown` +/// instead of requiring δ = 0. +#[test] +fn a_whole_result_failure_probability_without_values_per_evaluation_prunes_the_readout() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let query = memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: None, + }); + let logical = pass2(memo, &[entry(60_000)]); + let pruned = pruned_entry(&logical, own_candidate(readout)).expect("the readout is pruned"); + assert!(!pruned.causes.is_empty()); + for cause in &pruned.causes { + assert!( + matches!( + cause.root, + RootPruneReason::ValuesPerEvaluationUnknown { consumer, query: q } + if consumer == own_candidate(readout) && q == query + ), + "{cause:?}" + ); + } +} + +/// A readout another query gives a level is not pruned for a query whose +/// whole-result failure probability has no per-value one: Q0 (δ = 0.01, +/// values per evaluation unknown) and Q1 (no δ) read one KLL readout, and +/// the readout keeps Q1's level, live. Q0 reads its group, and the groups +/// below, only exactly: Q1's level says nothing about Q0's δ. Q1 may read +/// the readout and its state (only the rows below a build are exact for +/// both). +#[test] +fn a_readout_another_query_gives_a_level_survives_unknown_values_per_evaluation() { + let mut memo = Memo::new(&DataWorkload::default()); + let (window, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let q0 = memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: None, + }); + let q1 = query(&mut memo, readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + assert_eq!( + requirements_of(&logical.memo, readout), + vec![Some(RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error: 0.01, + failure_probability: 1.0, + })] + ); + let pruned = pruned_entry(&logical, own_candidate(readout)); + assert!(pruned.is_none(), "{pruned:?}"); + let exact_only = &logical.exact_only_groups_of_query; + for group in [readout, state, window] { + assert!(exact_only[q0].contains(&group), "G{}", group.0); + } + for group in [readout, state] { + assert!(!exact_only[q1].contains(&group), "G{}", group.0); + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/tests/rollup.rs new file mode 100644 index 000000000..75fd9e3b7 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/rollup.rs @@ -0,0 +1,43 @@ +//! Step 1: rollup from the finest common group, one file per concern: +//! compensating filters and matchers, the finest common group, and the +//! measures that roll up. +mod filters; +mod finest_common_group; +mod measures; + +use asap_types::ir::operator::arguments::Source; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::{NonASAPOp, Operator, Predicate, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::test_support::memo_builders::*; +use std::time::Duration; + +/// The instant vector `x` over `scan`: its latest sample per series. +pub(super) fn instant(memo: &mut Memo, scan: MemoGroupId) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(1), + kind: TimeRangeKind::Instant, + child: scan, + }), + ) +} + +/// A scan of the SQL table `flows(src, dst, bytes)` with `predicates`. +pub(super) fn flows(memo: &mut Memo, predicates: Vec>) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "flows".into(), + }, + predicates, + schema: Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("dst", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }), + ) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/rollup/filters.rs b/crates/stage1-pass2-reuse-candidates/src/tests/rollup/filters.rs new file mode 100644 index 000000000..ccd8eac9d --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/rollup/filters.rs @@ -0,0 +1,301 @@ +//! Compensating filters and matchers: a coarse group over fewer rows +//! rolls up a finer one through a `Filter` on its keys. +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::FieldDataType; + +use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; +use asap_types::summary_type::{ExactKind, SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::aggregate; +use asap_types::test_support::memo_fixtures::{compare, range}; + +use super::{flows, instant}; + +/// Concepts doc Example 1: F = `quantile by (job, host)(latency[5m])`, C = +/// `quantile by (host)(latency{job="api"}[5m])`. Equal definitions, the +/// narrowing `job ∈ {api}` on a key of F, {host} ⊆ {job, host}: C's state +/// gets `SummaryMerge(by host)(Filter(job = "api", F))`, and F is the +/// finest common group (nothing is synthesized). +#[test] +fn a_filtered_coarse_sketch_state_merges_the_finer_states_through_a_compensating_filter() { + let mut memo = Memo::new(&DataWorkload::default()); + let labels = ["job", "host"]; + let all = scan_group(&mut memo, "latency", &labels, vec![]); + let api = scan_group( + &mut memo, + "latency", + &labels, + vec![compare( + 2, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )], + ); + let fine_window = group(&mut memo, range(300_000, all)); + let coarse_window = group(&mut memo, range(300_000, api)); + let fine = sketch_state( + &mut memo, + fine_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2, 3]), + ); + let coarse = sketch_state( + &mut memo, + coarse_window, + SketchAlgorithm::Kll, + Reduction::by(vec![3]), + ); + let groups_before = memo.groups().len(); + for state in [fine, coarse] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let filter = group_of( + memo, + &Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + 0, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )), + child: fine, + }), + ) + .expect("Filter(job = \"api\", F)"); + assert!(candidates_of(memo, coarse).iter().any(|c| c.origin + == OriginOfCandidateForMemoGroup::Rollup + && c.expression.operator + == Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![filter], + group_by: Some(vec![1]), + }))); + // F reads nothing from C, and no finest common group was added: the + // memo grew by the readouts, the filter, and the chains only. + assert!(candidates_of(memo, fine) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::Rollup)); + assert!(groups_where(memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryAgg { .. }) + )) + .iter() + .all(|&g| g == fine || g == coarse || g.0 as usize >= groups_before)); +} + +/// A `quantile by (job)` over `status = "500"` rows does not roll up from +/// `quantile by (job, host)` over every row: `status` is no key of the +/// finer state, whose states mix both statuses. +#[test] +fn a_narrowing_on_a_column_that_is_no_key_gives_no_rollup() { + let mut memo = Memo::new(&DataWorkload::default()); + let labels = ["job", "host", "status"]; + let all = scan_group(&mut memo, "latency", &labels, vec![]); + let errors = scan_group( + &mut memo, + "latency", + &labels, + vec![compare( + 4, + CompareOpKind::Eq, + ScalarValue::Utf8("500".into()), + ExprSemantics::Promql, + )], + ); + let all_window = group(&mut memo, range(300_000, all)); + let errors_window = group(&mut memo, range(300_000, errors)); + let fine = sketch_state( + &mut memo, + all_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2, 3]), + ); + let coarse = sketch_state( + &mut memo, + errors_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2]), + ); + for state in [fine, coarse] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + assert!(candidates_of(&logical.memo, coarse).iter().all(|c| !c + .expression + .operator + .children() + .contains(&&fine))); +} + +/// A PromQL `!=` matcher compensates with PromQL's comparison: +/// `sum by (job)` over `x{instance!="i1"}` rolls up `sum by (job, +/// instance)` over `x` through `Filter(instance != "i1")` with PromQL +/// semantics (a series without the label matches), not a negated SQL +/// `IN` list. +#[test] +fn a_not_equal_matcher_compensates_with_promql_semantics() { + let mut memo = Memo::new(&DataWorkload::default()); + let not_equal = |column: usize| { + compare( + column, + CompareOpKind::Ne, + ScalarValue::Utf8("i1".into()), + ExprSemantics::Promql, + ) + }; + let all = scan_group(&mut memo, "x", &["job", "instance"], vec![]); + let not_i1 = scan_group(&mut memo, "x", &["job", "instance"], vec![not_equal(3)]); + let all_rows = instant(&mut memo, all); + let not_i1_rows = instant(&mut memo, not_i1); + let fine = group( + &mut memo, + aggregate( + Reduction::by(vec![2, 3]), + AggIntent::Sum { col: None }, + all_rows, + ), + ); + let coarse = group( + &mut memo, + aggregate( + Reduction::by(vec![2]), + AggIntent::Sum { col: None }, + not_i1_rows, + ), + ); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let filters = groups_where( + &logical.memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == fine), + ); + let [filter] = filters[..] else { + panic!("one compensating filter: {filters:?}"); + }; + assert_eq!( + logical.memo.groups()[filter.0 as usize].expression, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(not_equal(1)), + child: fine, + }) + ); +} + +/// A compensating filter compares with the semantics of the frontend of +/// the source it reads: `SUM(bytes) WHERE dst = 'x' GROUP BY src` over a +/// SQL table rolls up the exact sum state by (src, dst) through +/// `Filter(dst = 'x')` with SQL semantics, not PromQL's. +#[test] +fn a_compensating_filter_over_a_sql_table_has_sql_semantics() { + let mut memo = Memo::new(&DataWorkload::default()); + let all = flows(&mut memo, vec![]); + let x = flows( + &mut memo, + vec![Predicate(compare( + 1, + CompareOpKind::Eq, + ScalarValue::Utf8("x".into()), + ExprSemantics::Sql, + ))], + ); + let sum_state = |memo: &mut Memo, rows: MemoGroupId, keys: Vec| { + group( + memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: rows, + summary_type: FieldDataType::exact(ExactKind::Sum), + input: SummaryUpdate::column(ColumnRef::Named("bytes".into())), + reduction: Reduction::by(keys), + filter: None, + }), + ) + }; + let fine = sum_state(&mut memo, all, vec![0, 1]); + let coarse = sum_state(&mut memo, x, vec![0]); + for state in [fine, coarse] { + let root = group( + &mut memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + ); + query(&mut memo, root, Some(AccuracyTarget::exact())); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let filters = groups_where( + &logical.memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == fine), + ); + let [filter] = filters[..] else { + panic!("one compensating filter: {filters:?}"); + }; + assert_eq!( + logical.memo.groups()[filter.0 as usize].expression, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + 1, + CompareOpKind::Eq, + ScalarValue::Utf8("x".into()), + ExprSemantics::Sql + )), + child: fine, + }) + ); +} + +/// A rollup the memo rejects adds nothing: `COUNT(*) WHERE src = 'a'` (one +/// row, 0 when no row matches) does not roll up from `COUNT(*) GROUP BY +/// src` (a sum of counts, NULL over no row), and no compensating +/// `Filter(src = 'a')` group is left without a reader. +#[test] +fn a_rejected_rollup_leaves_no_compensating_filter() { + let mut memo = Memo::new(&DataWorkload::default()); + let all = flows(&mut memo, vec![]); + let a = flows( + &mut memo, + vec![Predicate(compare( + 0, + CompareOpKind::Eq, + ScalarValue::Utf8("a".into()), + ExprSemantics::Sql, + ))], + ); + let count_by = |memo: &mut Memo, rows: MemoGroupId, keys: Vec| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: rows, + reduction: Reduction::by(keys), + measures: vec![AggIntent::Count], + output_names: vec!["count".into()], + filters: vec![None], + having: None, + }), + ) + }; + let by_src = count_by(&mut memo, all, vec![0]); + let only_a = count_by(&mut memo, a, vec![]); + query(&mut memo, by_src, Some(AccuracyTarget::exact())); + query(&mut memo, only_a, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + assert!(candidates_of(memo, only_a) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::Rollup)); + let roots = memo.root_groups(); + let orphans: Vec = (0..memo.groups().len()) + .map(|index| MemoGroupId(index as u32)) + .filter(|g| !roots.contains(g) && memo.groups()[g.0 as usize].parent_groups().is_empty()) + .collect(); + assert!(orphans.is_empty(), "groups without a reader: {orphans:?}"); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/rollup/finest_common_group.rs b/crates/stage1-pass2-reuse-candidates/src/tests/rollup/finest_common_group.rs new file mode 100644 index 000000000..c141acc40 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/rollup/finest_common_group.rs @@ -0,0 +1,117 @@ +//! A cluster's finest common group, which no query asks for. +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; +use asap_types::ir::schema::FieldDataType; + +use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::state_grouped_by; +use asap_types::test_support::memo_fixtures::{compare, range}; + +/// Concepts doc Example 3: Q1 = `quantile by (host)(latency[5m])`, Q2 = +/// `quantile by (job)(latency{host="xxx"}[5m])`. Neither is finer, so the +/// finest common group G* = `SummaryAgg[KLL, by job, host]` over Q1's rows +/// is synthesized with one build per listed θ, Q1's state merges it by +/// host and Q2's merges `Filter(host = "xxx", G*)` by job. +#[test] +fn a_finest_common_group_no_query_asks_for_serves_both() { + let mut memo = Memo::new(&DataWorkload::default()); + let labels = ["job", "host"]; + let all = scan_group(&mut memo, "latency", &labels, vec![]); + let xxx = scan_group( + &mut memo, + "latency", + &labels, + vec![compare( + 3, + CompareOpKind::Eq, + ScalarValue::Utf8("xxx".into()), + ExprSemantics::Promql, + )], + ); + let all_window = group(&mut memo, range(300_000, all)); + let xxx_window = group(&mut memo, range(300_000, xxx)); + let by_host = sketch_state( + &mut memo, + all_window, + SketchAlgorithm::Kll, + Reduction::by(vec![3]), + ); + let by_job = sketch_state( + &mut memo, + xxx_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2]), + ); + for state in [by_host, by_job] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let finest = group_of( + memo, + &state_grouped_by( + FieldDataType::Sketch(SketchAlgorithm::Kll), + Reduction::by(vec![2, 3]), + all_window, + ), + ) + .expect("G*: KLL by job, host over Q1's rows"); + assert_eq!( + candidates_of(memo, finest) + .iter() + .filter(|c| c.expression.operator == memo.groups()[finest.0 as usize].expression) + .count(), + parameter_list(SketchAlgorithm::Kll).len() + ); + let merges = |state: MemoGroupId| -> Vec> { + candidates_of(memo, state) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect() + }; + let listed = parameter_list(SketchAlgorithm::Kll).len(); + // One merge per θ of G*'s builds. + assert_eq!( + merges(by_host), + vec![ + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![finest], + group_by: Some(vec![1]), + }); + listed + ] + ); + let filter = group_of( + memo, + &Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + 1, + CompareOpKind::Eq, + ScalarValue::Utf8("xxx".into()), + ExprSemantics::Promql, + )), + child: finest, + }), + ) + .expect("Filter(host = \"xxx\", G*)"); + assert_eq!(candidates_of(memo, filter).len(), listed); + assert_eq!( + merges(by_job), + vec![ + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![filter], + group_by: Some(vec![0]), + }); + listed + ] + ); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/rollup/measures.rs b/crates/stage1-pass2-reuse-candidates/src/tests/rollup/measures.rs new file mode 100644 index 000000000..94151cca7 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/rollup/measures.rs @@ -0,0 +1,111 @@ +//! The measures that roll up: a sum and a count as a sum. +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::schema::FieldDataType; + +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; +use asap_types::summary_type::ExactKind; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::range; +use asap_types::test_support::memo_fixtures::{aggregate, state_grouped_by}; + +use super::instant; + +/// Concepts doc Example 2, exact values: `sum by (job)` rolls up the +/// finalized `sum by (job, instance)` of the same per-series rows with a +/// sum over its sum column. +#[test] +fn a_coarse_exact_sum_rolls_up_the_finer_sum() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job", "instance"], vec![]); + let window = group(&mut memo, range(60_000, input)); + let rate = group( + &mut memo, + state_grouped_by( + FieldDataType::exact(ExactKind::Rate), + Reduction::PerEntity, + window, + ), + ); + let rows = group( + &mut memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: rate }), + ); + let sum_by = |memo: &mut Memo, keys: Vec, name: &str| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: rows, + reduction: Reduction::by(keys), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![name.into()], + filters: vec![None], + having: None, + }), + ) + }; + let fine = sum_by(&mut memo, vec![2, 3], "value"); + let coarse = sum_by(&mut memo, vec![2], "value"); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let rollups: Vec<_> = candidates_of(&logical.memo, coarse) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect(); + assert_eq!( + rollups, + vec![Operator::NonASAP(NonASAPOp::Aggregate { + child: fine, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec!["value".into()], + filters: vec![None], + having: None, + })] + ); +} + +/// A count rolls up as a sum of counts, under the count's own column name: +/// `count by (job)` of PromQL (its measure unnamed, `count`) rolls up the +/// finer `count by (job, instance)` with `sum` over its count column, named +/// `count` as the coarse group's. +#[test] +fn a_coarse_count_rolls_up_the_finer_count_as_a_sum() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job", "instance"], vec![]); + let rows = instant(&mut memo, input); + let fine = group( + &mut memo, + aggregate(Reduction::by(vec![2, 3]), AggIntent::Count, rows), + ); + let coarse = group( + &mut memo, + aggregate(Reduction::by(vec![2]), AggIntent::Count, rows), + ); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let rollups: Vec<_> = candidates_of(&logical.memo, coarse) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect(); + assert_eq!( + rollups, + vec![Operator::NonASAP(NonASAPOp::Aggregate { + child: fine, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec!["count".into()], + filters: vec![], + having: None, + })] + ); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/topk.rs b/crates/stage1-pass2-reuse-candidates/src/tests/topk.rs new file mode 100644 index 000000000..2577f535e --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/topk.rs @@ -0,0 +1,46 @@ +//! Step 2: top-k reuse. + +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; + +/// `topk(5, x)` reads `topk(10, x)`'s ranked rows: the 5 highest of the 10 +/// highest are the 5 highest. +#[test] +fn a_smaller_topk_reads_the_largest_topk_of_the_same_input() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job"], vec![]); + let topk = |memo: &mut Memo, k: usize| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: input, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::TopK { k }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ) + }; + let top5 = topk(&mut memo, 5); + let top10 = topk(&mut memo, 10); + query(&mut memo, top5, Some(AccuracyTarget::exact())); + query(&mut memo, top10, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let reuse: Vec<_> = candidates_of(&logical.memo, top5) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::TopKReuse) + .collect(); + assert_eq!(reuse.len(), 1); + assert_eq!(reuse[0].expression.operator.children(), vec![&top10]); + assert!(candidates_of(&logical.memo, top10) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::TopKReuse)); +} diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 6452f0bed..978ca7c0e 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -30,7 +30,7 @@ pub use report::{ /// `PhysicalASAPDAG` node per selected execution, its inputs from the /// selected edges, each node's `selected_physical_candidate` set and each /// materialized view node's `view_coverage` from its memo group -/// (`asap_types::stages::view_coverage`). +/// (`asap_types::plan::view_coverage`). /// `entries[q]` is query `q`'s workload entry; `data` the data workload /// with its bounds. /// @@ -46,5 +46,5 @@ pub fn select_plan( .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; let _ = (physical, entries, data); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage = asap_types::stages::view_coverage(memo, its group)") + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage = asap_types::plan::view_coverage(memo, its group)") } diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 8119895dc..ac00cf2f9 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -6,11 +6,10 @@ edition = "2021" # Shared vocabulary for the whole workspace: the L3 intent algebra (former # asap-ir) plus the L4 sketch-bound IR types (former asap-sketch, minus its -# execution logic — removed, no real implementor existed; see issue #190). +# execution logic — removed, no real implementor existed). # No internal deps. [dependencies] -# "rc" — OperatorNode child fields are Rc (issue #212, #222: -# shared sub-expressions), and Rc's Serialize/Deserialize impls live behind +# "rc" — OperatorNode child fields are Rc (shared sub-expressions), and Rc's Serialize/Deserialize impls live behind # this feature flag. A shared Rc still (de)serializes as an ordinary inline # value, once per reference, exactly like the old Box. serde = { version = "1", features = ["derive", "rc"] } diff --git a/crates/types/src/accuracy/default_metric.rs b/crates/types/src/accuracy/default_metric.rs new file mode 100644 index 000000000..f7d7c14ac --- /dev/null +++ b/crates/types/src/accuracy/default_metric.rs @@ -0,0 +1,212 @@ +//! The metric a query result is most naturally stated in, found by walking +//! the query's operators. + +use crate::ir::operator::AggIntent; +use crate::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; + +use super::AccuracyMetric; + +/// The metric a query's result is most naturally stated in, from the +/// computation at `root`: an L1-scaled absolute error for counts and top-k +/// frequencies, a relative error for distinct counts and L2 frequency +/// moments, and an absolute value error for anything else (sums, +/// arithmetic, rates, averages, extrema, entropy). `None` for a quantile +/// root, whose error is as naturally a rank as a relative one. +/// +/// Used only to size and to display: a target whose metric is not set +/// accepts a guarantee in any metric. +pub fn default_metric_for_root(root: &QueryRoot) -> Option { + match root { + QueryRoot::Operator(node) => node_metric(node), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node)) + | QueryRoot::Scalar(ScalarExpr::ScalarSubquery(node)) => node_metric(node), + QueryRoot::Scalar(_) => Some(AccuracyMetric::ABSOLUTE), + } +} + +/// The default metric of an intent's value, `None` for a quantile and for +/// intents without a listed default. +fn intent_metric(intent: &AggIntent) -> Option { + match intent { + AggIntent::Count | AggIntent::TopK { .. } => Some(AccuracyMetric::ABSOLUTE_STREAM_L1), + AggIntent::Cardinality { .. } | AggIntent::FrequencyL2 { .. } => { + Some(AccuracyMetric::Relative) + } + _ => None, + } +} + +/// The metric of `node`'s values, looking through operators that only +/// select, order or rename rows. +fn node_metric(node: &OperatorNode) -> Option { + let Operator::NonASAP(op) = &node.operator else { + return Some(AccuracyMetric::ABSOLUTE); + }; + match op { + NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::Dedup { child, .. } => node_metric(child), + NonASAPOp::Project { cols, child, .. } + if cols + .iter() + .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => + { + node_metric(child) + } + // A `CASE` choosing between columns and literals, e.g. SQL's + // empty-input NULL guard over a statistic cross-joined with an exact + // count: the metric of the columns its value branches read. + NonASAPOp::Project { .. } => (0..node.schema.fields.len()) + .map(|column| column_metric(node, column)) + .collect::>>() + .map(|metrics| match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => AccuracyMetric::ABSOLUTE, + }), + NonASAPOp::Aggregate { measures, .. } => { + let mut metrics = Vec::new(); + for intent in measures { + match intent { + AggIntent::Quantile { .. } | AggIntent::HistogramQuantile { .. } => { + return None + } + intent => { + metrics.push(intent_metric(intent).unwrap_or(AccuracyMetric::ABSOLUTE)) + } + } + } + // Measures with different default metrics share one value-unit + // statement. + Some(match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => AccuracyMetric::ABSOLUTE, + }) + } + _ => Some(AccuracyMetric::ABSOLUTE), + } +} + +/// The metric of `node`'s output `column`, following columns through +/// projections, `CASE` value branches, joins and operators that only +/// select, order or rename rows, down to an aggregate's measure. A group +/// key or any other expression is absolute. +pub(super) fn column_metric(node: &OperatorNode, column: usize) -> Option { + use crate::ir::operator::arguments::JoinKind; + let Operator::NonASAP(op) = &node.operator else { + return Some(AccuracyMetric::ABSOLUTE); + }; + match op { + NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::Dedup { child, .. } => column_metric(child, column), + NonASAPOp::Project { cols, child, .. } => match cols.get(column).map(|item| &item.expr) { + Some(ScalarExpr::Column(id)) => column_metric(child, *id), + Some(ScalarExpr::Case { + branches, + else_expr, + .. + }) => { + let mut metrics = Vec::new(); + for value in branches + .iter() + .map(|(_, then)| then) + .chain(else_expr.as_deref()) + { + match value { + // A literal, possibly cast (SQL's typed NULL). + ScalarExpr::Literal(_) => {} + ScalarExpr::Cast { expr, .. } + if matches!(**expr, ScalarExpr::Literal(_)) => {} + ScalarExpr::Column(id) => metrics.push(column_metric(child, *id)?), + _ => return Some(AccuracyMetric::ABSOLUTE), + } + } + Some(match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => AccuracyMetric::ABSOLUTE, + }) + } + _ => Some(AccuracyMetric::ABSOLUTE), + }, + NonASAPOp::Join { + kind: + JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + left, + right, + .. + } => { + let width = left.schema.fields.len(); + if column < width { + column_metric(left, column) + } else { + column_metric(right, column - width) + } + } + NonASAPOp::Aggregate { + reduction, + measures, + child, + .. + } => match aggregate_measure_at(node, reduction, measures, child, column) { + Some(AggIntent::Quantile { .. } | AggIntent::HistogramQuantile { .. }) => None, + Some(intent) => Some(intent_metric(intent).unwrap_or(AccuracyMetric::ABSOLUTE)), + None => Some(AccuracyMetric::ABSOLUTE), + }, + _ => Some(AccuracyMetric::ABSOLUTE), + } +} + +/// The measure the aggregate `node`'s output `column` holds, or `None` for +/// a group key, a kept label or `count_values`' synthesized label, as +/// [`aggregate_output_schema`](crate::ir::schema::aggregate_schema::aggregate_output_schema) +/// lays the columns out. A per-entity reduction replaces its one measure's +/// input column in place and keeps every other column; a cross-series one +/// emits its kept labels first, then one column per measure, with a +/// `count_values` label (unless an earlier column has its name) before its +/// count. +fn aggregate_measure_at<'a>( + node: &OperatorNode, + reduction: &crate::ir::operator::arguments::Reduction, + measures: &'a [AggIntent], + child: &OperatorNode, + column: usize, +) -> Option<&'a AggIntent> { + use crate::ir::operator::arguments::Reduction; + use crate::ir::scalar::{resolve_column_ref, ColumnRef}; + let fields = &node.schema.fields; + match reduction { + Reduction::PerEntity => { + let [measure] = measures else { return None }; + let value = match measure.input_cols().first() { + Some(input) => *input, + None => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?, + }; + (column == value).then_some(measure) + } + Reduction::Reduce(by) => { + let mut next = if by.is_without() { + fields.len().checked_sub(measures.len())? + } else { + by.len() + }; + for measure in measures { + if let AggIntent::CountValues { label } = measure { + if !by.is_without() + && !fields[..next.min(fields.len())] + .iter() + .any(|f| f.name == *label) + { + next += 1; + } + } + if next == column { + return Some(measure); + } + next += 1; + } + None + } + } +} diff --git a/crates/types/src/accuracy/guarantee.rs b/crates/types/src/accuracy/guarantee.rs deleted file mode 100644 index e432b41df..000000000 --- a/crates/types/src/accuracy/guarantee.rs +++ /dev/null @@ -1,751 +0,0 @@ -//! Machine-readable accuracy guarantees for finalized post-ASAP values -//! (issue #172). -//! -//! A selected post-ASAP plan used to carry no statement about the error of -//! the value it produces: every approximate layer was sized from its own -//! [`AccuracyTarget`] as if its input were exact, so an approximate parent -//! could silently consume an approximate child. This module is the -//! *vocabulary* that fixes that — the typed metric, the symbolic bound and -//! failure-probability expressions, the provenance trail, and the typed -//! rejection reasons. The *algebra* that composes these (the built-in -//! conservative rules and budget allocation) and the deployment's measured -//! accuracy (`AccuracyEvidence`) live one layer up in `asap_accuracy`: -//! this crate defines the shapes, the planning crates decide. -//! -//! ## What a guarantee says -//! -//! [`AccuracyGuarantee`] is attached to a finalized, caller-visible value — -//! [`crate::ir::OperatorNode::guarantee`] on a `SummaryEstimate` evaluation, an -//! exact accumulator, or a kept pre-ASAP sub-DAG — never to raw summary -//! state (a `SummaryAgg` sketch node carries `None`; its evaluation carries the -//! guarantee). Its statement is: -//! -//! ```text -//! Pr[ err_metric(estimate, truth) > error ] <= failure_probability -//! ``` -//! -//! for each output value separately ([`AccuracyScope::PerValue`]), where -//! `err_metric` is fixed by [`AccuracyMetric`]: an absolute error (in value -//! units or in multiples of a stream norm), a relative error, or a rank -//! error. Metrics are **not** interchangeable: converting between them -//! needs an explicit rule with evidence, never an implicit "compare the -//! epsilons". [`AccuracyGuarantee`] shares its outer shape, [`AccuracySpec`], -//! with the requirement it is checked against, [`AccuracyTarget`]. -//! -//! ## Why expressions, not numbers -//! -//! [`BoundExpr`]/[`ProbabilityExpr`] are tiny serializable expression DAGs -//! rather than bare `f64`s so a planning-time guarantee can reference a -//! statistic it does not have (a group count, a stream's L1 norm) and stay -//! honestly *unknown* until something instantiates it — a deployment's own -//! cardinality estimate, or a runtime posterior observation (issue #239). -//! [`BoundExpr::evaluate`] returns `None`, never `0`, for such a bound; -//! "unknown" and "zero" are different answers and a fail-closed planner -//! treats them differently. -//! -//! ## What is deliberately *not* here -//! -//! No `CorrectnessPolicy`-style enum: [`AccuracyTarget`] remains the one -//! authoritative requirement type, and [`AccuracyError`] is the typed reason -//! a candidate failed against it. No independence assumptions: the only -//! probability combinator is the union bound. - -use serde::{Deserialize, Serialize}; - -use super::target::{AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget}; - -/// A symbolic, serializable error-bound expression. Non-negative real -/// arithmetic only — there is no subtraction, so a bound can never be -/// tightened by construction, only by evaluating a known statistic. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "op", rename_all = "snake_case")] -pub enum BoundExpr { - /// Exactly zero error (deterministic exact computation). - Zero, - /// A resolved numeric bound in the metric's own normalization. - Constant { value: f64 }, - /// `Σ terms` — the additive composition rule. - Sum { terms: Vec }, - /// `Π factors` — e.g. the relative-error cross term, or a normalized - /// bound times the (possibly unknown) statistic it is normalized by. - Product { factors: Vec }, - /// `factor · inner` — an explicitly registered Lipschitz constant. - Scaled { factor: f64, inner: Box }, - /// `max(terms)` — exact max/min over bounded inputs. - Max { terms: Vec }, - /// A statistic this bound needs but nothing has supplied yet. Evaluates - /// to `None`, never `0`: an unknown quantity is not a small one. - Unknown { statistic: String }, -} - -impl BoundExpr { - #[cfg(test)] - fn optimistic_floor(&self) -> Self { - match self { - Self::Unknown { .. } => Self::Zero, - Self::Sum { terms } => Self::Sum { - terms: terms.iter().map(Self::optimistic_floor).collect(), - }, - Self::Product { factors } => Self::Product { - factors: factors.iter().map(Self::optimistic_floor).collect(), - }, - Self::Scaled { factor, inner } => Self::Scaled { - factor: *factor, - inner: Box::new(inner.optimistic_floor()), - }, - Self::Max { terms } => Self::Max { - terms: terms.iter().map(Self::optimistic_floor).collect(), - }, - other => other.clone(), - } - } - - /// See [`AccuracyGuarantee::bind`]. - pub(crate) fn bind(&self, value: &dyn Fn(&str) -> Option) -> Self { - let all = |terms: &[BoundExpr]| terms.iter().map(|t| t.bind(value)).collect(); - match self { - BoundExpr::Zero | BoundExpr::Constant { .. } => self.clone(), - BoundExpr::Sum { terms } => BoundExpr::Sum { terms: all(terms) }, - BoundExpr::Product { factors } => BoundExpr::Product { - factors: all(factors), - }, - BoundExpr::Max { terms } => BoundExpr::Max { terms: all(terms) }, - BoundExpr::Scaled { factor, inner } => BoundExpr::Scaled { - factor: *factor, - inner: Box::new(inner.bind(value)), - }, - BoundExpr::Unknown { statistic } => { - value(statistic).map_or_else(|| self.clone(), |value| BoundExpr::Constant { value }) - } - } - } - - /// Whether evaluation depends on a statistic not yet supplied. - #[cfg(any(test, feature = "test-support"))] - pub fn has_unknown(&self) -> bool { - match self { - BoundExpr::Zero | BoundExpr::Constant { .. } => false, - BoundExpr::Sum { terms } - | BoundExpr::Product { factors: terms } - | BoundExpr::Max { terms } => terms.iter().any(BoundExpr::has_unknown), - BoundExpr::Scaled { inner, .. } => inner.has_unknown(), - BoundExpr::Unknown { .. } => true, - } - } - - /// Numeric value of this bound, or `None` if any [`BoundExpr::Unknown`] - /// leaf is reachable. - pub fn evaluate(&self) -> Option { - let value = match self { - BoundExpr::Zero => Some(0.0), - BoundExpr::Constant { value } => value.is_finite().then_some(*value), - BoundExpr::Sum { terms } => terms.iter().map(BoundExpr::evaluate).sum(), - BoundExpr::Product { factors } => factors.iter().map(BoundExpr::evaluate).product(), - BoundExpr::Scaled { factor, inner } => factor - .is_finite() - .then_some(*factor) - .zip(inner.evaluate()) - .map(|(factor, bound)| factor * bound), - BoundExpr::Max { terms } => terms - .iter() - .map(BoundExpr::evaluate) - .try_fold(0.0_f64, |acc, t| t.map(|t| acc.max(t))), - BoundExpr::Unknown { .. } => None, - }?; - (value.is_finite() && value >= 0.0).then_some(value) - } - - /// `true` iff this bound is structurally zero (every leaf is - /// [`BoundExpr::Zero`], or a `Product`/`Scaled` contains a zero factor). - /// Distinct from `evaluate() == Some(0.0)` only in that it never - /// depends on floating-point evaluation. - pub fn is_zero(&self) -> bool { - match self { - BoundExpr::Zero => true, - BoundExpr::Constant { value } => *value == 0.0, - BoundExpr::Sum { terms } | BoundExpr::Max { terms } => { - terms.iter().all(BoundExpr::is_zero) - } - BoundExpr::Product { factors } => factors.iter().any(BoundExpr::is_zero), - BoundExpr::Scaled { factor, inner } => *factor == 0.0 || inner.is_zero(), - BoundExpr::Unknown { .. } => false, - } - } -} - -/// A symbolic, serializable failure-probability expression. The only -/// combinator over several events is the union bound — the default model -/// never assumes independence between sketch errors. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "op", rename_all = "snake_case")] -pub enum ProbabilityExpr { - /// The guarantee is deterministic. - Zero, - /// A resolved probability in `[0, 1]`. - Constant { value: f64 }, - /// `min(1, Σ terms)` — Boole's inequality over the listed events. - UnionBound { terms: Vec }, - /// `min(1, count · inner)` — the union bound over `count` events that - /// each fail with probability at most `inner` (e.g. one per input row of - /// an exact aggregation). `count` is a [`BoundExpr`] so it may be an - /// [`BoundExpr::Unknown`] statistic. - Scaled { - count: BoundExpr, - inner: Box, - }, - /// A probability nothing has supplied yet — same stance as - /// [`BoundExpr::Unknown`]. - Unknown { statistic: String }, -} - -impl ProbabilityExpr { - #[cfg(test)] - fn optimistic_floor(&self) -> Self { - match self { - Self::Unknown { .. } => Self::Zero, - Self::UnionBound { terms } => Self::UnionBound { - terms: terms.iter().map(Self::optimistic_floor).collect(), - }, - Self::Scaled { count, inner } => Self::Scaled { - count: count.optimistic_floor(), - inner: Box::new(inner.optimistic_floor()), - }, - other => other.clone(), - } - } - - /// Whether evaluation depends on a statistic not yet supplied. - #[cfg(any(test, feature = "test-support"))] - pub(crate) fn has_unknown(&self) -> bool { - match self { - ProbabilityExpr::Zero | ProbabilityExpr::Constant { .. } => false, - ProbabilityExpr::UnionBound { terms } => terms.iter().any(ProbabilityExpr::has_unknown), - ProbabilityExpr::Scaled { count, inner } => count.has_unknown() || inner.has_unknown(), - ProbabilityExpr::Unknown { .. } => true, - } - } - - /// Numeric value clamped to `[0, 1]`, or `None` if any unknown leaf is - /// reachable. - pub fn evaluate(&self) -> Option { - let raw = match self { - ProbabilityExpr::Zero => 0.0, - ProbabilityExpr::Constant { value } if (0.0..=1.0).contains(value) => *value, - ProbabilityExpr::Constant { .. } => return None, - ProbabilityExpr::UnionBound { terms } => terms - .iter() - .map(ProbabilityExpr::evaluate) - .sum::>()?, - ProbabilityExpr::Scaled { count, inner } => count.evaluate()? * inner.evaluate()?, - ProbabilityExpr::Unknown { .. } => return None, - }; - raw.is_finite().then(|| raw.clamp(0.0, 1.0)) - } - - /// See [`AccuracyGuarantee::bind`]. - pub(crate) fn bind(&self, value: &dyn Fn(&str) -> Option) -> Self { - match self { - ProbabilityExpr::Zero | ProbabilityExpr::Constant { .. } => self.clone(), - ProbabilityExpr::UnionBound { terms } => ProbabilityExpr::UnionBound { - terms: terms.iter().map(|t| t.bind(value)).collect(), - }, - ProbabilityExpr::Scaled { count, inner } => ProbabilityExpr::Scaled { - count: count.bind(value), - inner: Box::new(inner.bind(value)), - }, - ProbabilityExpr::Unknown { statistic } => value(statistic) - .map_or_else(|| self.clone(), |value| ProbabilityExpr::Constant { value }), - } - } - - /// `true` iff this probability is structurally zero. - pub fn is_zero(&self) -> bool { - match self { - ProbabilityExpr::Zero => true, - ProbabilityExpr::Constant { value } => *value == 0.0, - ProbabilityExpr::UnionBound { terms } => terms.iter().all(ProbabilityExpr::is_zero), - ProbabilityExpr::Scaled { count, inner } => count.is_zero() || inner.is_zero(), - ProbabilityExpr::Unknown { .. } => false, - } - } -} - -/// How a parent operator consumes its inputs' values — the shape an -/// `propagate` rule is registered against. `#[non_exhaustive]` -/// for the same reason [`AccuracyMetric`] is. -#[non_exhaustive] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "op", rename_all = "snake_case")] -pub enum CompositionOperator { - /// Relative division with runtime finite/nonzero/range checks. For operand - /// bounds a,b the ratio bound is (a+b)/(1-b), with b < 1. - CheckedRelativeDivision, - /// An approximate summary built over its inputs' (approximate) values - /// — the sketch-over-sketch case. Its own `local` guarantee composes - /// with the inputs' under a same-metric rule. - ApproximateAggregate, - /// A deterministic transformation with an explicitly registered global - /// Lipschitz constant: `B_out ≤ constant · B_in + B_local`. The planner - /// never derives `constant` itself; only a caller that has proved it - /// may construct this operator. - Lipschitz { constant: f64 }, - /// An exact sum over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. - ExactSum, - /// An exact arithmetic mean over approximate values. For absolute-value - /// input guarantees, the output bound is the largest per-value bound. - ExactAverage, - /// An exact max/min over inputs carrying absolute-value guarantees — bounds the returned - /// *value* (`max` of the input bounds) but does not identify which key - /// is the true winner. - ExactExtremum, - /// Exact division of two relative-value estimates. For numerator bound - /// `a` and denominator bound `b < 1`, the output bound is - /// `(a + b) / (1 - b)`. - ExactDivision, - /// Exact multiplication of two estimates. For absolute errors `e_x`, - /// `e_y` the output bound is `|x|·e_y + |y|·e_x + e_x·e_y`, requiring - /// finite operand domains; for relative errors `a`, `b` it is - /// `(1 + a)(1 + b) − 1`. - ExactProduct, - /// PromQL `rate`: reset correction plus range-boundary extrapolation. - CounterRate, - /// PromQL `irate`: reset-aware slope over the final two samples. - InstantCounterRate, - /// PromQL `increase`: extrapolated reset-corrected increase over a range. - CounterIncrease, - /// A top-k selection over approximate inputs. Without a separated-margin - /// certificate the default model leaves membership accuracy unknown. - TopKSelection, -} - -/// One entry in a [`AccuracyGuarantee`]'s provenance trail — enough for a -/// reader to reconstruct *why* the bound is what it is without re-running -/// the planner. `#[non_exhaustive]` so runtime evidence (issue #239) and -/// deployment-specific sources can be appended later. -#[non_exhaustive] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum GuaranteeSource { - /// Deterministic exact computation — zero error by construction. - Exact { - /// What made it exact (e.g. `"ExactAggregate(Sum)"`, - /// `"RetainedExact"`). - reason: String, - }, - /// The target this evaluation's sketch was sized against. - AccuracyTarget { target: AccuracyTarget }, - /// The concrete sketch a evaluation's local guarantee was derived from. - SketchEvaluation { - algorithm: String, - /// Stable estimator/analysis contract used to derive this guarantee. - #[serde(default)] - contract: String, - params: serde_json::Value, - query: String, - }, - /// A composed input's own guarantee, carried verbatim so the trail is - /// self-contained. `input_index` is the input's position in the - /// composition (0-based, in the parent's child order). - ChildGuarantee { - input_index: usize, - guarantee: Box, - }, - /// The propagation rule that produced this guarantee from its inputs. - CompositionStep { - operator: CompositionOperator, - /// Stable rule name (e.g. `"additive_union_bound"`). - rule: String, - }, - /// The budget split that produced this layer's local target — present - /// only when an equal budget split re-sized a layer. - BudgetAllocation { - allocator: String, - layer: usize, - layer_count: usize, - local_target: AccuracyTarget, - end_to_end_target: AccuracyTarget, - }, - /// A statistic the bound needs but nothing supplied — the reason a - /// [`BoundExpr::Unknown`]/[`ProbabilityExpr::Unknown`] leaf exists. - UnavailableStatistic { statistic: String }, - /// Query-time evidence (issue #239's posterior bounds). Never produced - /// at planning time; reserved so a runtime can append its observation - /// to the same trail instead of inventing a parallel one. - RuntimeObservation { - source: String, - detail: serde_json::Value, - }, -} - -/// The statistic an unbound number of values per result is left as -/// ([`AccuracyGuarantee::over_scope`]); nothing binds it, so the -/// failure probability stays unknown. -pub const VALUES_PER_RESULT: &str = "values_per_result"; - -/// The machine-readable accuracy statement attached to a finalized -/// post-ASAP value — see the module docs for its semantics. Its -/// [`AccuracySpec`] is symbolic, for plan selection to bind; a guarantee a -/// sketch or a composition produces covers each output value separately -/// ([`AccuracyScope::PerValue`]). -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct AccuracyGuarantee { - pub spec: AccuracySpec, - /// The explanation trail: why the bound is what it is. No planning - /// decision reads it. - pub provenance: Vec, -} - -impl AccuracyGuarantee { - /// A per-value guarantee: error at most `error` in `metric`, failing - /// with probability at most `failure_probability`. - pub fn new( - metric: AccuracyMetric, - error: BoundExpr, - failure_probability: ProbabilityExpr, - provenance: Vec, - ) -> Self { - Self { - spec: AccuracySpec { - accuracy_metric: metric, - error, - failure_probability, - accuracy_scope: AccuracyScope::PerValue, - }, - provenance, - } - } - - /// This guarantee with every [`BoundExpr::Unknown`] (and - /// [`ProbabilityExpr::Unknown`]) leaf whose statistic `value` supplies - /// replaced by that value. Plan selection binds statistics this way; - /// a statistic `value` does not supply stays unknown. - pub fn bind(&self, value: &dyn Fn(&str) -> Option) -> Self { - let mut bound = self.clone(); - bound.spec.error = self.spec.error.bind(value); - bound.spec.failure_probability = self.spec.failure_probability.bind(value); - bound - } - - /// Optimistic lower bound (tests only): unknown non-negative - /// contributions count as zero, so it cannot certify a candidate; the - /// original guarantee remains unresolved. - #[cfg(test)] - fn optimistic_floor(&self) -> Self { - let mut floor = self.clone(); - floor.spec.error = self.spec.error.optimistic_floor(); - floor.spec.failure_probability = self.spec.failure_probability.optimistic_floor(); - floor - } - - /// An unresolved bound or failure probability is not a certificate. - #[cfg(any(test, feature = "test-support"))] - pub fn has_unknown(&self) -> bool { - self.spec.error.has_unknown() || self.spec.failure_probability.has_unknown() - } - - /// The zero-error, zero-failure guarantee of a deterministic exact - /// computation. `metric` is [`AccuracyMetric::ABSOLUTE`]: an exact value - /// is exact under every metric, and absolute error is the one every - /// same-metric rule accepts as a zero input. - pub fn exact(reason: impl Into) -> Self { - Self::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Zero, - ProbabilityExpr::Zero, - vec![GuaranteeSource::Exact { - reason: reason.into(), - }], - ) - } - - /// This guarantee stated over `scope`, for a result of - /// `values_per_result` values (`G`, `None` when unbound): a per-value - /// failure probability `δ` becomes `n · δ` by the union bound over the - /// `n` values the scope covers ([`AccuracyScope::max_values_covered`]), - /// whatever their dependence. The error is unchanged. A guarantee that - /// never fails (`δ = 0`) holds for every value at once; an unbound `n` - /// (the whole result of an unbound `G`) leaves the failure probability - /// symbolic in [`VALUES_PER_RESULT`]. A guarantee over any other scope - /// is returned as it is. Plan selection uses it to check a target over - /// `scope`. - pub fn over_scope(&self, scope: AccuracyScope, values_per_result: Option) -> Self { - let mut over = self.clone(); - // Only a per-value statement extends to more values by the union - // bound: one over some groups says nothing about the others, so it - // is returned as it is and misses a scope it does not cover. - if self.spec.accuracy_scope != AccuracyScope::PerValue || scope == AccuracyScope::PerValue { - return over; - } - over.spec.accuracy_scope = scope; - if !self.spec.failure_probability.is_zero() { - over.spec.failure_probability = ProbabilityExpr::Scaled { - count: scope.max_values_covered(values_per_result).map_or_else( - || BoundExpr::Unknown { - statistic: VALUES_PER_RESULT.into(), - }, - |values| BoundExpr::Constant { - value: values as f64, - }, - ), - inner: Box::new(self.spec.failure_probability.clone()), - }; - } - over - } - - /// `true` iff this guarantee promises zero error with certainty. - pub fn is_exact(&self) -> bool { - self.spec.error.is_zero() && self.spec.failure_probability.is_zero() - } - - /// How many approximate sketch evaluations contributed to this value — - /// `1` for a plain evaluation, `0` for an exact value, and the transitive - /// count through every [`GuaranteeSource::ChildGuarantee`] for a - /// composition. - #[cfg(any(test, feature = "test-support"))] - pub fn approximate_layer_count(&self) -> usize { - self.provenance - .iter() - .map(|source| match source { - GuaranteeSource::SketchEvaluation { .. } => 1, - GuaranteeSource::ChildGuarantee { guarantee, .. } => { - guarantee.approximate_layer_count() - } - _ => 0, - }) - .sum() - } -} - -/// Why an accuracy check rejected a candidate. Typed, serializable, and -/// carried through to DAG export so a rejection is as inspectable as a -/// selection. Never a reason to "treat the child as exact". -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, thiserror::Error)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum AccuracyError { - /// No registered propagation rule covers this operator over these - /// input metrics (or this local metric). - #[error( - "unsupported accuracy composition: {operator:?} over inputs {input_metrics:?} \ - with local {local_metric:?} — {reason}" - )] - UnsupportedComposition { - operator: CompositionOperator, - input_metrics: Vec, - local_metric: Option, - reason: String, - }, - /// An approximate input carries no guarantee at all, so nothing can be - /// composed over it. - #[error("input {input_index} of {operator:?} carries no accuracy guarantee")] - MissingInputGuarantee { - operator: CompositionOperator, - input_index: usize, - }, - /// The composed guarantee does not satisfy the applicable - /// [`AccuracyTarget`]. `bound`/`failure_probability` are the evaluated - /// values when known. - #[error( - "composed guarantee ({metric:?}, bound {bound:?}, failure probability \ - {failure_probability:?}) does not satisfy {target:?}" - )] - TargetNotSatisfied { - metric: AccuracyMetric, - bound: Option, - failure_probability: Option, - target: AccuracyTarget, - }, - /// No budget allocation could make the composition legal under the - /// end-to-end target. - #[error("no legal accuracy-budget allocation for {target:?} across {layer_count} layers")] - NoLegalAllocation { - target: AccuracyTarget, - layer_count: usize, - }, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn unknown_statistic_never_evaluates_to_zero() { - let b = BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: "input_row_count".into(), - }, - ], - }; - assert_eq!(b.evaluate(), None); - assert!(!b.is_zero()); - let p = ProbabilityExpr::Scaled { - count: BoundExpr::Unknown { - statistic: "input_row_count".into(), - }, - inner: Box::new(ProbabilityExpr::Constant { value: 0.01 }), - }; - assert_eq!(p.evaluate(), None); - } - - #[test] - fn optimistic_floor_preserves_known_contributions_without_certifying() { - let guarantee = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE_STREAM_L1, - BoundExpr::Sum { - terms: vec![ - BoundExpr::Constant { value: 0.1 }, - BoundExpr::Unknown { - statistic: "collision".into(), - }, - ], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - ProbabilityExpr::Constant { value: 0.02 }, - ProbabilityExpr::Unknown { - statistic: "failure".into(), - }, - ], - }, - vec![], - ); - assert!(guarantee.has_unknown()); - assert_eq!(guarantee.spec.error.evaluate(), None); - let floor = guarantee.optimistic_floor(); - assert_eq!(floor.spec.error.evaluate(), Some(0.1)); - assert_eq!(floor.spec.failure_probability.evaluate(), Some(0.02)); - assert!(!floor.has_unknown()); - assert!(guarantee.has_unknown()); - } - - #[test] - fn invalid_numeric_leaves_fail_closed() { - assert_eq!(BoundExpr::Constant { value: -0.1 }.evaluate(), None); - assert_eq!( - BoundExpr::Scaled { - factor: -1.0, - inner: Box::new(BoundExpr::Constant { value: 0.1 }), - } - .evaluate(), - None - ); - assert_eq!(ProbabilityExpr::Constant { value: -0.1 }.evaluate(), None); - assert_eq!(ProbabilityExpr::Constant { value: 1.1 }.evaluate(), None); - } - - #[test] - fn union_bound_sums_and_clamps() { - let p = ProbabilityExpr::UnionBound { - terms: vec![ - ProbabilityExpr::Constant { value: 0.7 }, - ProbabilityExpr::Constant { value: 0.6 }, - ], - }; - assert_eq!(p.evaluate(), Some(1.0)); - } - - #[test] - fn exact_guarantee_is_zero_layers() { - let g = AccuracyGuarantee::exact("test"); - assert!(g.is_exact()); - assert_eq!(g.approximate_layer_count(), 0); - } - - #[test] - fn guarantee_round_trips_through_json() { - let g = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE_STREAM_L1, - BoundExpr::Sum { - terms: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Scaled { - factor: 2.0, - inner: Box::new(BoundExpr::Constant { value: 0.005 }), - }, - ], - }, - ProbabilityExpr::UnionBound { - terms: vec![ProbabilityExpr::Constant { value: 0.01 }], - }, - vec![GuaranteeSource::CompositionStep { - operator: CompositionOperator::Lipschitz { constant: 2.0 }, - rule: "lipschitz".into(), - }], - ); - let json = serde_json::to_value(&g).unwrap(); - assert_eq!(json["spec"]["accuracy_metric"], "absolute_stream_l1"); - assert_eq!(json["spec"]["accuracy_scope"], "per_value"); - assert_eq!(json["spec"]["error"]["op"], "sum"); - let back: AccuracyGuarantee = serde_json::from_value(json).unwrap(); - assert_eq!(back, g); - } - - /// A per-value guarantee over the groups holding at least 5% of the - /// records fails with at most 20 times its per-value probability, even - /// when the number of values the result returns is unknown: at most 20 - /// groups can hold 5% each. Fewer values bound it lower. The error is - /// unchanged, and a scope the guarantee already covers keeps it as it is. - #[test] - fn a_record_share_scope_union_bounds_over_one_over_share_groups() { - let per_value = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: 0.05 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let five_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.05); - let over = per_value.over_scope(five_percent, None); - assert_eq!(over.spec.accuracy_scope, five_percent); - assert_eq!(over.spec.error, per_value.spec.error); - assert!((over.spec.failure_probability.evaluate().unwrap() - 0.02).abs() < 1e-12); - let few = per_value.over_scope(five_percent, Some(4)); - assert!((few.spec.failure_probability.evaluate().unwrap() - 0.004).abs() < 1e-12); - assert_eq!( - per_value.over_scope(AccuracyScope::PerValue, None), - per_value - ); - let whole = per_value.over_scope(AccuracyScope::WholeResult, Some(10)); - assert_eq!(whole.over_scope(five_percent, None), whole); - // A statement over the 5% groups says nothing about groups between - // 1% and 5%, nor about the whole result: it is not rescaled. - let one_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.01); - assert_eq!(over.over_scope(one_percent, None), over); - assert_eq!(over.over_scope(AccuracyScope::WholeResult, Some(10)), over); - assert_eq!( - per_value - .over_scope(AccuracyScope::WholeResult, None) - .spec - .failure_probability - .evaluate(), - None - ); - } - - /// A supplied statistic replaces its `Unknown` leaves, in the bound and - /// in the failure probability; any other statistic stays unknown. - #[test] - fn binding_a_statistic_replaces_only_its_unknown_leaves() { - let rows = || BoundExpr::Unknown { - statistic: "input_row_count".into(), - }; - let g = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Product { - factors: vec![BoundExpr::Constant { value: 0.01 }, rows()], - }, - ProbabilityExpr::Scaled { - count: rows(), - inner: Box::new(ProbabilityExpr::Constant { value: 1e-4 }), - }, - vec![], - ); - let bound = g.bind(&|statistic| (statistic == "input_row_count").then_some(10.0)); - assert_eq!(bound.spec.error.evaluate(), Some(0.1)); - assert!((bound.spec.failure_probability.evaluate().unwrap() - 1e-3).abs() < 1e-12); - let unbound = g.bind(&|_| None); - assert_eq!(unbound, g); - } -} diff --git a/crates/types/src/accuracy/guarantee/mod.rs b/crates/types/src/accuracy/guarantee/mod.rs new file mode 100644 index 000000000..1341df7ad --- /dev/null +++ b/crates/types/src/accuracy/guarantee/mod.rs @@ -0,0 +1,120 @@ +//! Machine-readable accuracy guarantees: the vocabulary for what error a +//! finalized, caller-visible value has — the typed metric +//! ([`AccuracyGuarantee`]) and the bound and failure-probability +//! expressions ([`BoundExpr`], [`ProbabilityExpr`]). The theoretical +//! guarantee of a summary (`theoretical_guarantee`), each operator's error +//! contract and the deployment's measured accuracy (`AccuracyEvidence`) +//! live one layer up in `asap_accuracy`: this crate defines the shapes, the +//! planning crates decide. +//! +//! ## What a guarantee says +//! +//! [`AccuracyGuarantee`] describes a finalized, caller-visible value — a +//! `SummaryEstimate` evaluation or an exact accumulator — never raw summary +//! state. Its statement is: +//! +//! ```text +//! Pr[ err_metric(estimate, truth) > error ] <= failure_probability +//! ``` +//! +//! for each output value separately ([`AccuracyScope::PerValue`]), where +//! `err_metric` is fixed by [`AccuracyMetric`]: an absolute error (in value +//! units or in multiples of a stream norm), a relative error, or a rank +//! error. Metrics are **not** interchangeable: converting between them +//! needs an explicit rule with evidence, never an implicit "compare the +//! epsilons". [`AccuracyGuarantee`] shares its outer shape, [`AccuracySpec`], +//! with the requirement it is checked against, [`AccuracyTarget`](super::AccuracyTarget). +//! +//! A failure probability may be *unknown* ([`ProbabilityExpr::Unknown`], +//! HLL's): [`ProbabilityExpr::evaluate`] returns `None`, never `0`, for it; +//! "unknown" and "zero" are different answers and a fail-closed planner +//! treats them differently. + +use serde::{Deserialize, Serialize}; + +use super::target::{AccuracyMetric, AccuracyScope, AccuracySpec}; + +/// A serializable error bound, in the metric's own normalization. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum BoundExpr { + /// A resolved numeric bound. + Constant { value: f64 }, +} + +impl BoundExpr { + /// Numeric value of this bound; `None` when it is negative or not + /// finite. + pub fn evaluate(&self) -> Option { + let BoundExpr::Constant { value } = self; + (value.is_finite() && *value >= 0.0).then_some(*value) + } +} + +/// A serializable failure-probability expression. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum ProbabilityExpr { + /// The guarantee is deterministic. + Zero, + /// A resolved probability in `[0, 1]`. + Constant { value: f64 }, + /// A probability nothing has supplied (HLL's). Evaluates to `None`, + /// never `0`: an unknown probability is not a small one. + Unknown { statistic: String }, +} + +impl ProbabilityExpr { + /// Numeric value in `[0, 1]`; `None` when it is unknown or out of + /// range. + pub fn evaluate(&self) -> Option { + match self { + ProbabilityExpr::Zero => Some(0.0), + ProbabilityExpr::Constant { value } if (0.0..=1.0).contains(value) => Some(*value), + ProbabilityExpr::Constant { .. } | ProbabilityExpr::Unknown { .. } => None, + } + } +} + +/// The machine-readable accuracy statement of a finalized post-ASAP value +/// — see the module docs for its semantics. A guarantee a sketch produces +/// covers each output value separately ([`AccuracyScope::PerValue`]). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AccuracyGuarantee { + pub spec: AccuracySpec, +} + +impl AccuracyGuarantee { + /// A per-value guarantee: error at most `error` in `metric`, failing + /// with probability at most `failure_probability`. + pub fn new( + metric: AccuracyMetric, + error: BoundExpr, + failure_probability: ProbabilityExpr, + ) -> Self { + Self { + spec: AccuracySpec { + accuracy_metric: metric, + error, + failure_probability, + accuracy_scope: AccuracyScope::PerValue, + }, + } + } + + /// The zero-error, zero-failure guarantee of a deterministic exact + /// computation. `metric` is [`AccuracyMetric::ABSOLUTE`]: an exact value + /// is exact under every metric, and absolute error is the one every + /// same-metric rule accepts as a zero input. + pub fn exact() -> Self { + Self::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Constant { value: 0.0 }, + ProbabilityExpr::Zero, + ) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/accuracy/guarantee/tests.rs b/crates/types/src/accuracy/guarantee/tests.rs new file mode 100644 index 000000000..efd297ee3 --- /dev/null +++ b/crates/types/src/accuracy/guarantee/tests.rs @@ -0,0 +1,34 @@ +use super::*; + +/// An unknown failure probability never evaluates to zero. +#[test] +fn an_unknown_failure_probability_never_evaluates_to_zero() { + let unknown = ProbabilityExpr::Unknown { + statistic: "hll_failure_probability".into(), + }; + assert_eq!(unknown.evaluate(), None); + assert_eq!(ProbabilityExpr::Zero.evaluate(), Some(0.0)); +} + +#[test] +fn invalid_numeric_leaves_fail_closed() { + assert_eq!(BoundExpr::Constant { value: -0.1 }.evaluate(), None); + assert_eq!(BoundExpr::Constant { value: f64::NAN }.evaluate(), None); + assert_eq!(ProbabilityExpr::Constant { value: -0.1 }.evaluate(), None); + assert_eq!(ProbabilityExpr::Constant { value: 1.1 }.evaluate(), None); +} + +#[test] +fn guarantee_round_trips_through_json() { + let g = AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE_STREAM_L1, + BoundExpr::Constant { value: 0.01 }, + ProbabilityExpr::Constant { value: 0.01 }, + ); + let json = serde_json::to_value(&g).unwrap(); + assert_eq!(json["spec"]["accuracy_metric"], "absolute_stream_l1"); + assert_eq!(json["spec"]["accuracy_scope"], "per_value"); + assert_eq!(json["spec"]["error"]["op"], "constant"); + let back: AccuracyGuarantee = serde_json::from_value(json).unwrap(); + assert_eq!(back, g); +} diff --git a/crates/types/src/accuracy/mod.rs b/crates/types/src/accuracy/mod.rs index 0bcce29fb..6780f97d5 100644 --- a/crates/types/src/accuracy/mod.rs +++ b/crates/types/src/accuracy/mod.rs @@ -1,16 +1,13 @@ -//! #511 §2.2: accuracy targets and the error metrics they are stated in +//! Accuracy targets and the error metrics they are stated in //! ([`AccuracyTarget`], [`AccuracyMetric`]), and machine-readable accuracy //! guarantees ([`AccuracyGuarantee`]). //! //! - `target` — accuracy targets and the error metrics they are stated in. //! - `guarantee` — machine-readable accuracy guarantees. +mod default_metric; mod guarantee; mod target; -pub use guarantee::{ - AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, - ProbabilityExpr, VALUES_PER_RESULT, -}; -pub use target::{ - default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, -}; +pub use default_metric::default_metric_for_root; +pub use guarantee::{AccuracyGuarantee, BoundExpr, ProbabilityExpr}; +pub use target::{AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale}; diff --git a/crates/types/src/accuracy/target.rs b/crates/types/src/accuracy/target.rs deleted file mode 100644 index 1b3562729..000000000 --- a/crates/types/src/accuracy/target.rs +++ /dev/null @@ -1,824 +0,0 @@ -//! Accuracy requirements and the error metrics they are stated in (#690). -//! -//! A requirement and a guarantee share one outer shape, [`AccuracySpec`]: -//! an error metric, an error ε, a failure probability δ and a scope. A -//! requirement ([`AccuracyTarget`]) holds concrete numbers and may leave its -//! metric unset; a guarantee ([`crate::accuracy::AccuracyGuarantee`]) -//! always states its metric and holds symbolic expressions that plan -//! selection binds. A target without a metric accepts a guarantee in any -//! metric that meets its ε, δ and scope. - -use serde::{Deserialize, Serialize}; - -use crate::ir::operator::AggIntent; -use crate::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; - -/// The stream norm an [`AccuracyMetric::Absolute`] error is stated in -/// multiples of. A scaled error `ε` means `|estimate − truth| ≤ ε · ‖f‖`, -/// where `f` is the frequency vector of the one stream the estimate's -/// summaries read: `‖f‖₁` for Count-Min, `‖f‖₂` for Count Sketch. Two scaled -/// errors compare directly only over the same stream; restating one in value -/// units multiplies it by that stream's norm, a statistic plan selection -/// binds from the data workload (`stream_l1_norm[source]`, -/// `stream_l2_norm[source]`). Since `‖f‖₂ ≤ ‖f‖₁`, an L2-scaled error meets -/// an L1-scaled requirement of the same stream with the same ε. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum NormScale { - StreamL1, - StreamL2, -} - -/// Which error quantity an accuracy statement bounds. `#[non_exhaustive]`: -/// a metric is added only when a primitive needs a notion these cannot -/// express (e.g. a false-positive rate), and downstream matches must not -/// assume the list is closed. -#[non_exhaustive] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(into = "MetricName", from = "MetricName")] -pub enum AccuracyMetric { - /// `|estimate − truth| ≤ ε`: in the value's own units when `scale` is - /// `None`, or in multiples of a stream norm ([`NormScale`]). - Absolute { scale: Option }, - /// `|estimate − truth| ≤ ε · |truth|`, e.g. DDSketch's α or a distinct - /// count's relative error. - Relative, - /// The returned value's rank in the input multiset is within `ε · n` of - /// the requested rank (a quantile's KLL ε). Says nothing about how far - /// the returned value is from the true quantile value. - Rank, -} - -impl AccuracyMetric { - /// Absolute error in the value's own units. - pub const ABSOLUTE: Self = Self::Absolute { scale: None }; - /// Absolute error in multiples of the stream's L1 norm (Count-Min). - pub const ABSOLUTE_STREAM_L1: Self = Self::Absolute { - scale: Some(NormScale::StreamL1), - }; - /// Absolute error in multiples of the stream's L2 norm (Count Sketch). - pub const ABSOLUTE_STREAM_L2: Self = Self::Absolute { - scale: Some(NormScale::StreamL2), - }; -} - -/// The wire name of an [`AccuracyMetric`]. -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -enum MetricName { - Absolute, - AbsoluteStreamL1, - AbsoluteStreamL2, - Relative, - Rank, -} - -impl From for MetricName { - fn from(metric: AccuracyMetric) -> Self { - match metric { - AccuracyMetric::Absolute { scale: None } => Self::Absolute, - AccuracyMetric::Absolute { - scale: Some(NormScale::StreamL1), - } => Self::AbsoluteStreamL1, - AccuracyMetric::Absolute { - scale: Some(NormScale::StreamL2), - } => Self::AbsoluteStreamL2, - AccuracyMetric::Relative => Self::Relative, - AccuracyMetric::Rank => Self::Rank, - } - } -} - -impl From for AccuracyMetric { - fn from(name: MetricName) -> Self { - match name { - MetricName::Absolute => Self::ABSOLUTE, - MetricName::AbsoluteStreamL1 => Self::ABSOLUTE_STREAM_L1, - MetricName::AbsoluteStreamL2 => Self::ABSOLUTE_STREAM_L2, - MetricName::Relative => Self::Relative, - MetricName::Rank => Self::Rank, - } - } -} - -/// Which output values one (ε, δ) statement covers *simultaneously*. -/// -/// A scope bounds only the failure probability: a per-value guarantee with -/// failure probability `δ` holds over the `n` values a scope covers with -/// `n · δ` by the union bound ([`Self::max_values_covered`]), whatever their -/// dependence. Not `Eq`: a record share is an `f64`. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AccuracyScope { - /// Each output value (one series or row, at one evaluation) is within ε - /// of its true value with probability ≥ 1 − δ, separately. Values may - /// fail in different runs; nothing is said about all of them at once. - /// Sketches and composition produce guarantees of this scope. - PerValue, - /// Every value in one evaluation's returned result is within ε - /// simultaneously, with probability ≥ 1 − δ. The default of a target: a - /// target at a query root applies to the entire returned result. - WholeResult, - /// Every group (result row) holding at least this share of the records - /// the query aggregates in one evaluation is within ε simultaneously, - /// with probability ≥ 1 − δ; smaller groups are not covered. The share - /// is a fraction in (0, 1] (0.05 for 5%). When a row reads several - /// aggregations, its group must hold the share of each one's records. - /// At most `⌊1 / share⌋` groups can qualify, whatever the number of - /// groups. JSON: `{"groups_with_record_share_at_least": 0.05}`. - GroupsWithRecordShareAtLeast(#[serde(deserialize_with = "record_share")] f64), -} - -/// Relative tolerance for `⌊1 / share⌋`, so a share such as 0.05 whose -/// reciprocal computes just below an integer counts that integer. -const RECORD_SHARE_ROUNDING_TOLERANCE: f64 = 1e-9; - -/// A record share, rejected unless it is a fraction in (0, 1]. -fn record_share<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { - let share = f64::deserialize(deserializer)?; - if share > 0.0 && share <= 1.0 { - Ok(share) - } else { - Err(serde::de::Error::custom(format!( - "a record share is a fraction in (0, 1], got {share}" - ))) - } -} - -impl AccuracyScope { - /// How many values one evaluation's statement over this scope covers at - /// most, given the `values_per_result` (`G`) the result returns, `None` - /// when unbound: one for [`Self::PerValue`], `G` for - /// [`Self::WholeResult`], and `min(⌊1 / share⌋, G)` for - /// [`Self::GroupsWithRecordShareAtLeast`], which is bound even when `G` - /// is not. Sizing divides a target's failure probability by it, and the - /// guarantee check multiplies a per-value one by it, so the strictest of - /// several scopes is the one that covers the most values. - pub fn max_values_covered(self, values_per_result: Option) -> Option { - match self { - Self::PerValue => Some(1), - Self::WholeResult => values_per_result, - // A share outside (0, 1] (possible only through the Rust API; - // JSON rejects it) bounds nothing, so its target stays unmet. - Self::GroupsWithRecordShareAtLeast(share) if !(share > 0.0 && share <= 1.0) => None, - Self::GroupsWithRecordShareAtLeast(share) => { - // More than `1 / share` groups cannot each hold `share` of - // the records. Rounding up near an integer keeps `1 / 0.05` - // at 20 when it computes just below; it only enlarges the - // count, which is conservative. - let qualifying = - (share.recip() * (1.0 + RECORD_SHARE_ROUNDING_TOLERANCE)).floor() as u64; - Some(values_per_result.map_or(qualifying, |values| values.min(qualifying))) - } - } - } -} - -/// The outer shape of an accuracy guarantee -/// ([`crate::accuracy::AccuracyGuarantee`]): -/// `Pr[err_metric(estimate, truth) > error] ≤ failure_probability` over the -/// values `accuracy_scope` names. An [`AccuracyTarget`] has the same fields, with -/// concrete numbers, an optional metric and `max_error` for `error`: a target -/// states the largest error a query accepts, a guarantee the error bound -/// (an expression over the plan's parameters) its plan achieves. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct AccuracySpec { - pub accuracy_metric: AccuracyMetric, - /// ε: the error bound, in `accuracy_metric`'s units. - pub error: E, - /// δ. - pub failure_probability: P, - pub accuracy_scope: AccuracyScope, -} - -/// An accuracy requirement, as a workload, a caller or a frontend states -/// it: `Pr[err_metric(estimate, truth) > max_error] ≤ failure_probability` over -/// the values `accuracy_scope` names. -/// -/// `accuracy_metric: None` means the user did not set one: plan selection -/// then accepts a guarantee in any metric whose ε, δ and scope meet this -/// target's, and reports the metric it accepted. A set metric accepts only -/// guarantees in that metric (or convertible to it with evidence). -/// `failure_probability: None` imposes no δ requirement (sizing then uses -/// the planner's default δ). Exact is `max_error = 0`, -/// `failure_probability = Some(0)`, whatever the metric. -/// -/// Built with [`Self::exact`] or [`Self::max_error`] plus the optional -/// builders [`Self::failure_probability`], [`Self::accuracy_metric`] and -/// [`Self::accuracy_scope`]. -/// -/// JSON: `{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": -/// 0.001, "accuracy_scope": "whole_result"}`; every field but `max_error` may be -/// omitted (`accuracy_scope` defaults to `whole_result`). Exact is -/// `{"max_error": 0.0, "failure_probability": 0.0}`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct AccuracyTarget { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub accuracy_metric: Option, - /// ε: the largest error the query accepts, in the metric's units. - pub max_error: f64, - /// δ. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub failure_probability: Option, - #[serde(default = "whole_result")] - pub accuracy_scope: AccuracyScope, -} - -fn whole_result() -> AccuracyScope { - AccuracyScope::WholeResult -} - -impl AccuracyTarget { - /// No approximation permitted. - pub const fn exact() -> Self { - Self { - accuracy_metric: None, - max_error: 0.0, - failure_probability: Some(0.0), - accuracy_scope: AccuracyScope::WholeResult, - } - } - - /// Error at most `max_error` over the whole result, with no δ - /// requirement and no metric set. - pub const fn max_error(max_error: f64) -> Self { - Self { - accuracy_metric: None, - max_error, - failure_probability: None, - accuracy_scope: AccuracyScope::WholeResult, - } - } - - /// This target, met with probability at least `1 − failure_probability`. - pub const fn failure_probability(self, failure_probability: f64) -> Self { - Self { - failure_probability: Some(failure_probability), - ..self - } - } - - /// This target with its error stated in `accuracy_metric`. - pub const fn accuracy_metric(self, accuracy_metric: AccuracyMetric) -> Self { - Self { - accuracy_metric: Some(accuracy_metric), - ..self - } - } - - /// This target over `accuracy_scope`. - pub const fn accuracy_scope(self, accuracy_scope: AccuracyScope) -> Self { - Self { - accuracy_scope, - ..self - } - } - - /// Whether this target permits no approximation. - pub fn is_exact(&self) -> bool { - self.max_error == 0.0 && self.failure_probability == Some(0.0) - } -} - -/// The metric a query's result is most naturally stated in, from the -/// computation at `root`: an L1-scaled absolute error for counts and top-k -/// frequencies, a relative error for distinct counts and L2 frequency -/// moments, and an absolute value error for anything else (sums, -/// arithmetic, rates, averages, extrema, entropy). `None` for a quantile -/// root, whose error is as naturally a rank as a relative one. -/// -/// Used only to size and to display: a target whose metric is not set -/// accepts a guarantee in any metric (#690). -pub fn default_metric_for_root(root: &QueryRoot) -> Option { - match root { - QueryRoot::Operator(node) => node_metric(node), - QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node)) - | QueryRoot::Scalar(ScalarExpr::ScalarSubquery(node)) => node_metric(node), - QueryRoot::Scalar(_) => Some(AccuracyMetric::ABSOLUTE), - } -} - -/// The default metric of an intent's value, `None` for a quantile and for -/// intents without a listed default. -fn intent_metric(intent: &AggIntent) -> Option { - match intent { - AggIntent::Count | AggIntent::TopK { .. } => Some(AccuracyMetric::ABSOLUTE_STREAM_L1), - AggIntent::Cardinality { .. } | AggIntent::FrequencyL2 { .. } => { - Some(AccuracyMetric::Relative) - } - _ => None, - } -} - -/// The metric of `node`'s values, looking through operators that only -/// select, order or rename rows. -fn node_metric(node: &OperatorNode) -> Option { - let Operator::NonASAP(op) = &node.operator else { - return Some(AccuracyMetric::ABSOLUTE); - }; - match op { - NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } - | NonASAPOp::Dedup { child, .. } => node_metric(child), - NonASAPOp::Project { cols, child, .. } - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - node_metric(child) - } - // A `CASE` choosing between columns and literals, e.g. SQL's - // empty-input NULL guard over a statistic cross-joined with an exact - // count: the metric of the columns its value branches read. - NonASAPOp::Project { .. } => (0..node.schema.fields.len()) - .map(|column| column_metric(node, column)) - .collect::>>() - .map(|metrics| match metrics.split_first() { - Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, - _ => AccuracyMetric::ABSOLUTE, - }), - NonASAPOp::Aggregate { measures, .. } => { - let mut metrics = Vec::new(); - for intent in measures { - match intent { - AggIntent::Quantile { .. } | AggIntent::HistogramQuantile { .. } => { - return None - } - intent => { - metrics.push(intent_metric(intent).unwrap_or(AccuracyMetric::ABSOLUTE)) - } - } - } - // Measures with different default metrics share one value-unit - // statement. - Some(match metrics.split_first() { - Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, - _ => AccuracyMetric::ABSOLUTE, - }) - } - _ => Some(AccuracyMetric::ABSOLUTE), - } -} - -/// The metric of `node`'s output `column`, following columns through -/// projections, `CASE` value branches, joins and operators that only -/// select, order or rename rows, down to an aggregate's measure. A group -/// key or any other expression is absolute. -fn column_metric(node: &OperatorNode, column: usize) -> Option { - use crate::ir::operator::arguments::JoinKind; - let Operator::NonASAP(op) = &node.operator else { - return Some(AccuracyMetric::ABSOLUTE); - }; - match op { - NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } - | NonASAPOp::Dedup { child, .. } => column_metric(child, column), - NonASAPOp::Project { cols, child, .. } => match cols.get(column).map(|item| &item.expr) { - Some(ScalarExpr::Column(id)) => column_metric(child, *id), - Some(ScalarExpr::Case { - branches, - else_expr, - .. - }) => { - let mut metrics = Vec::new(); - for value in branches - .iter() - .map(|(_, then)| then) - .chain(else_expr.as_deref()) - { - match value { - // A literal, possibly cast (SQL's typed NULL). - ScalarExpr::Literal(_) => {} - ScalarExpr::Cast { expr, .. } - if matches!(**expr, ScalarExpr::Literal(_)) => {} - ScalarExpr::Column(id) => metrics.push(column_metric(child, *id)?), - _ => return Some(AccuracyMetric::ABSOLUTE), - } - } - Some(match metrics.split_first() { - Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, - _ => AccuracyMetric::ABSOLUTE, - }) - } - _ => Some(AccuracyMetric::ABSOLUTE), - }, - NonASAPOp::Join { - kind: - JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - left, - right, - .. - } => { - let width = left.schema.fields.len(); - if column < width { - column_metric(left, column) - } else { - column_metric(right, column - width) - } - } - NonASAPOp::Aggregate { - reduction, - measures, - child, - .. - } => match aggregate_measure_at(node, reduction, measures, child, column) { - Some(AggIntent::Quantile { .. } | AggIntent::HistogramQuantile { .. }) => None, - Some(intent) => Some(intent_metric(intent).unwrap_or(AccuracyMetric::ABSOLUTE)), - None => Some(AccuracyMetric::ABSOLUTE), - }, - _ => Some(AccuracyMetric::ABSOLUTE), - } -} - -/// The measure the aggregate `node`'s output `column` holds, or `None` for -/// a group key, a kept label or `count_values`' synthesized label, as -/// [`aggregate_output_schema`](crate::ir::schema::aggregate_schema::aggregate_output_schema) -/// lays the columns out. A per-entity reduction replaces its one measure's -/// input column in place and keeps every other column; a cross-series one -/// emits its kept labels first, then one column per measure, with a -/// `count_values` label (unless an earlier column has its name) before its -/// count. -fn aggregate_measure_at<'a>( - node: &OperatorNode, - reduction: &crate::ir::operator::arguments::Reduction, - measures: &'a [AggIntent], - child: &OperatorNode, - column: usize, -) -> Option<&'a AggIntent> { - use crate::ir::operator::arguments::Reduction; - use crate::ir::scalar::{resolve_column_ref, ColumnRef}; - let fields = &node.schema.fields; - match reduction { - Reduction::PerEntity => { - let [measure] = measures else { return None }; - let value = match measure.input_cols().first() { - Some(input) => *input, - None => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?, - }; - (column == value).then_some(measure) - } - Reduction::Reduce(by) => { - let mut next = if by.is_without() { - fields.len().checked_sub(measures.len())? - } else { - by.len() - }; - for measure in measures { - if let AggIntent::CountValues { label } = measure { - if !by.is_without() - && !fields[..next.min(fields.len())] - .iter() - .any(|f| f.name == *label) - { - next += 1; - } - } - if next == column { - return Some(measure); - } - next += 1; - } - None - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::operator::{Reduction, Source}; - use crate::ir::schema::{DataType, Field, Schema}; - use std::rc::Rc; - - fn node(op: NonASAPOp) -> Rc { - Rc::new(OperatorNode::new(Operator::NonASAP(op)).expect("fixture derives a schema")) - } - - fn aggregate(measures: Vec) -> QueryRoot { - let scan = node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - }); - QueryRoot::Operator(node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures, - output_names: vec![], - filters: vec![], - having: None, - child: scan, - })) - } - - fn quantile() -> AggIntent { - AggIntent::Quantile { col: None, q: 0.99 } - } - - /// A target reads and writes as a plain struct: every field but - /// `max_error` may be omitted (no metric, no δ, the whole result), an unset - /// metric or δ writes nothing, and the old named forms (`"Exact"`, - /// `{"Epsilon": ε}`, `{"EpsilonDelta": …}`) and unknown fields are - /// rejected. - #[test] - fn accuracy_target_json_is_a_plain_struct() { - use serde_json::json; - let read = |value: serde_json::Value| serde_json::from_value::(value); - assert_eq!( - read(json!({"max_error": 0.01})).unwrap(), - AccuracyTarget::max_error(0.01) - ); - assert_eq!( - read(json!({"max_error": 0.0, "failure_probability": 0.0})).unwrap(), - AccuracyTarget::exact() - ); - let rank = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - assert_eq!( - read( - json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.001}) - ) - .unwrap(), - rank - ); - assert_eq!( - serde_json::to_value(&rank).unwrap(), - json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.001, - "accuracy_scope": "whole_result"}) - ); - assert_eq!( - serde_json::to_value(AccuracyTarget::max_error(0.01)).unwrap(), - json!({"max_error": 0.01, "accuracy_scope": "whole_result"}) - ); - for old in [ - json!("Exact"), - json!({"Epsilon": 0.01}), - json!({"EpsilonDelta": {"epsilon": 0.01, "delta": 0.001}}), - json!({"max_error": 0.1, "typo": true}), - ] { - assert!(read(old.clone()).is_err(), "{old}"); - } - } - - /// Metrics have stable wire names. - #[test] - fn metric_wire_names() { - for (metric, name) in [ - (AccuracyMetric::ABSOLUTE, "absolute"), - (AccuracyMetric::ABSOLUTE_STREAM_L1, "absolute_stream_l1"), - (AccuracyMetric::ABSOLUTE_STREAM_L2, "absolute_stream_l2"), - (AccuracyMetric::Relative, "relative"), - (AccuracyMetric::Rank, "rank"), - ] { - assert_eq!(serde_json::to_value(metric).unwrap(), name); - assert_eq!( - serde_json::from_value::(name.into()).unwrap(), - metric - ); - } - } - - /// A per-entity count (`count_over_time`) keeps every label column and - /// replaces the sample value in place: the value column carries the - /// count's metric, and a label after it is absolute. - #[test] - fn a_per_entity_count_states_its_value_column_not_its_last_column() { - let scan = node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, false), - ], - 0, - vec![], - ), - }); - let count = node(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![], - having: None, - child: scan, - }); - assert_eq!(count.schema.fields[1].name, "value"); - assert_eq!( - column_metric(&count, 1), - Some(AccuracyMetric::ABSOLUTE_STREAM_L1) - ); - assert_eq!(column_metric(&count, 2), Some(AccuracyMetric::ABSOLUTE)); - } - - /// A root's default metric follows its computation: counts and top-k - /// frequencies are L1-scaled absolute errors, distinct counts and L2 - /// frequency moments relative ones, anything else (a sum or an entropy - /// among them) absolute value errors. - #[test] - fn root_metric_defaults_follow_the_query_intent() { - let metric = |intent: AggIntent| default_metric_for_root(&aggregate(vec![intent])).unwrap(); - assert_eq!(metric(AggIntent::Count), AccuracyMetric::ABSOLUTE_STREAM_L1); - assert_eq!( - metric(AggIntent::Sum { col: None }), - AccuracyMetric::ABSOLUTE - ); - assert_eq!( - metric(AggIntent::TopK { k: 5 }), - AccuracyMetric::ABSOLUTE_STREAM_L1 - ); - assert_eq!( - metric(AggIntent::Cardinality { cols: vec![] }), - AccuracyMetric::Relative - ); - assert_eq!( - metric(AggIntent::FrequencyL2 { col: None }), - AccuracyMetric::Relative - ); - assert_eq!( - metric(AggIntent::FrequencyEntropy { col: None }), - AccuracyMetric::ABSOLUTE - ); - assert_eq!( - metric(AggIntent::Avg { col: None }), - AccuracyMetric::ABSOLUTE - ); - assert_eq!( - default_metric_for_root(&QueryRoot::Scalar(ScalarExpr::literal_f64(1.0))), - Some(AccuracyMetric::ABSOLUTE) - ); - } - - /// SQL's empty-input NULL guard (a statistic cross-joined with an exact - /// row count, and `CASE WHEN count = 0 THEN NULL ELSE statistic END`) - /// keeps the statistic's metric: the root's value is the statistic's - /// column. A `CASE` whose value branches read columns of different - /// metrics, or any other expression, is absolute. - #[test] - fn root_metric_looks_through_a_null_guard_to_its_statistic() { - use crate::ir::operator::arguments::JoinKind; - use crate::ir::scalar::{CompareOpKind, ScalarValue}; - use crate::ir::{Predicate, ProjectItem}; - let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { col: None }]) else { - unreachable!() - }; - let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count]) else { - unreachable!() - }; - let joined = node(NonASAPOp::Join { - kind: JoinKind::Cross, - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - left: l2, - right: count, - }); - let guard = |otherwise: ScalarExpr| { - QueryRoot::Operator(node(NonASAPOp::Project { - cols: vec![ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::Case { - operand: None, - branches: vec![( - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(1)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), - semantics: crate::ir::ExprSemantics::Sql, - }, - ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Literal(ScalarValue::Null)), - to: DataType::Float64, - try_cast: false, - }, - )], - else_expr: Some(Box::new(otherwise)), - }, - }], - child: joined.clone(), - })) - }; - let metric = |root: QueryRoot| default_metric_for_root(&root).unwrap(); - assert_eq!( - metric(guard(ScalarExpr::Column(0))), - AccuracyMetric::Relative - ); - // The count's column, or arithmetic over the statistic. - assert_eq!( - metric(guard(ScalarExpr::Column(1))), - AccuracyMetric::ABSOLUTE_STREAM_L1 - ); - assert_eq!( - metric(guard(ScalarExpr::Negative { - expr: Box::new(ScalarExpr::Column(0)), - semantics: crate::ir::ExprSemantics::Sql, - })), - AccuracyMetric::ABSOLUTE - ); - } - - /// A quantile root has no default metric: its error is as naturally a - /// rank as a relative one, and a target that sets none accepts either. - /// It is not an input error. - #[test] - fn a_quantile_root_has_no_default_metric() { - assert_eq!(default_metric_for_root(&aggregate(vec![quantile()])), None); - } - - /// A target on the groups holding at least a share of the records reads - /// and writes its share as a fraction; a share outside (0, 1] (say 5 - /// meant as 5%, or 0, which would cover every group) is bad input. - #[test] - fn a_record_share_scope_reads_its_share_as_a_fraction() { - use serde_json::json; - let target = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_scope(AccuracyScope::GroupsWithRecordShareAtLeast(0.05)); - let wire = json!({"max_error": 0.05, "failure_probability": 0.05, - "accuracy_scope": {"groups_with_record_share_at_least": 0.05}}); - assert_eq!(serde_json::to_value(&target).unwrap(), wire); - assert_eq!( - serde_json::from_value::(wire).unwrap(), - target - ); - for share in [json!(0.0), json!(-0.1), json!(5.0), json!(1.5)] { - let bad = json!({"max_error": 0.05, - "accuracy_scope": {"groups_with_record_share_at_least": share}}); - assert!( - serde_json::from_value::(bad).is_err(), - "{share}" - ); - } - } - - /// How many values one statement covers at most: one for a per-value - /// scope, the `G` values of the result for the whole result (unbound when - /// `G` is), and at most `⌊1 / share⌋` groups for a record share, since - /// more groups cannot each hold that share. So a 1% scope covers 100, - /// a 5% scope 20, fewer when the result has fewer values. - #[test] - fn a_scope_covers_at_most_one_over_its_share_groups() { - use AccuracyScope::*; - assert_eq!(PerValue.max_values_covered(None), Some(1)); - assert_eq!(WholeResult.max_values_covered(None), None); - assert_eq!(WholeResult.max_values_covered(Some(625)), Some(625)); - assert_eq!( - GroupsWithRecordShareAtLeast(0.05).max_values_covered(None), - Some(20) - ); - assert_eq!( - GroupsWithRecordShareAtLeast(0.01).max_values_covered(None), - Some(100) - ); - assert_eq!( - GroupsWithRecordShareAtLeast(0.03).max_values_covered(None), - Some(33) - ); - assert_eq!( - GroupsWithRecordShareAtLeast(1.0).max_values_covered(None), - Some(1) - ); - for invalid in [0.0, -0.1, 1.5, f64::NAN] { - assert_eq!( - GroupsWithRecordShareAtLeast(invalid).max_values_covered(None), - None, - "{invalid}" - ); - } - assert_eq!( - GroupsWithRecordShareAtLeast(0.01).max_values_covered(Some(4)), - Some(4) - ); - } - - /// Exact means zero error with certainty, whatever the metric; an - /// ε of zero without a δ is not exact. - #[test] - fn exact_targets() { - assert!(AccuracyTarget::exact().is_exact()); - assert_eq!(AccuracyTarget::exact().accuracy_metric, None); - assert!(AccuracyTarget::exact() - .accuracy_metric(AccuracyMetric::Rank) - .is_exact()); - assert!(!AccuracyTarget::max_error(0.0).is_exact()); - assert_eq!( - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .failure_probability, - Some(0.001) - ); - assert_eq!( - AccuracyTarget::max_error(0.01).accuracy_scope, - AccuracyScope::WholeResult - ); - } -} diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs new file mode 100644 index 000000000..18d908020 --- /dev/null +++ b/crates/types/src/accuracy/target/mod.rs @@ -0,0 +1,299 @@ +//! Accuracy requirements and the error metrics they are stated in. +//! +//! A requirement and a guarantee share one outer shape, [`AccuracySpec`]: +//! an error metric, an error ε, a failure probability δ and a scope. A +//! requirement ([`AccuracyTarget`]) holds concrete numbers and may leave its +//! metric unset; a guarantee ([`crate::accuracy::AccuracyGuarantee`]) +//! always states its metric and holds symbolic expressions that plan +//! selection binds. A target without a metric accepts a guarantee in any +//! metric that meets its ε, δ and scope. + +use serde::{Deserialize, Serialize}; + +/// The stream norm an [`AccuracyMetric::Absolute`] error is stated in +/// multiples of. A scaled error `ε` means `|estimate − truth| ≤ ε · ‖f‖`, +/// where `f` is the frequency vector of the one stream the estimate's +/// summaries read: `‖f‖₁` for Count-Min, `‖f‖₂` for Count Sketch. Two scaled +/// errors compare directly only over the same stream; restating one in value +/// units multiplies it by that stream's norm, a statistic plan selection +/// binds from the data workload (`stream_l1_norm[source]`, +/// `stream_l2_norm[source]`). Since `‖f‖₂ ≤ ‖f‖₁`, an L2-scaled error meets +/// an L1-scaled requirement of the same stream with the same ε. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum NormScale { + StreamL1, + StreamL2, +} + +/// Which error quantity an accuracy statement bounds. `#[non_exhaustive]`: +/// a metric is added only when a primitive needs a notion these cannot +/// express (e.g. a false-positive rate), and downstream matches must not +/// assume the list is closed. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(into = "MetricName", from = "MetricName")] +pub enum AccuracyMetric { + /// `|estimate − truth| ≤ ε`: in the value's own units when `scale` is + /// `None`, or in multiples of a stream norm ([`NormScale`]). + Absolute { scale: Option }, + /// `|estimate − truth| ≤ ε · |truth|`, e.g. DDSketch's α or a distinct + /// count's relative error. + Relative, + /// The returned value's rank in the input multiset is within `ε · n` of + /// the requested rank (a quantile's KLL ε). Says nothing about how far + /// the returned value is from the true quantile value. + Rank, +} + +impl AccuracyMetric { + /// Absolute error in the value's own units. + pub const ABSOLUTE: Self = Self::Absolute { scale: None }; + /// Absolute error in multiples of the stream's L1 norm (Count-Min). + pub const ABSOLUTE_STREAM_L1: Self = Self::Absolute { + scale: Some(NormScale::StreamL1), + }; + /// Absolute error in multiples of the stream's L2 norm (Count Sketch). + pub const ABSOLUTE_STREAM_L2: Self = Self::Absolute { + scale: Some(NormScale::StreamL2), + }; + /// Every metric, for a contract or a target read in each of them. + pub const ALL: [Self; 5] = [ + Self::ABSOLUTE, + Self::ABSOLUTE_STREAM_L1, + Self::ABSOLUTE_STREAM_L2, + Self::Relative, + Self::Rank, + ]; +} + +/// The wire name of an [`AccuracyMetric`]. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum MetricName { + Absolute, + AbsoluteStreamL1, + AbsoluteStreamL2, + Relative, + Rank, +} + +impl From for MetricName { + fn from(metric: AccuracyMetric) -> Self { + match metric { + AccuracyMetric::Absolute { scale: None } => Self::Absolute, + AccuracyMetric::Absolute { + scale: Some(NormScale::StreamL1), + } => Self::AbsoluteStreamL1, + AccuracyMetric::Absolute { + scale: Some(NormScale::StreamL2), + } => Self::AbsoluteStreamL2, + AccuracyMetric::Relative => Self::Relative, + AccuracyMetric::Rank => Self::Rank, + } + } +} + +impl From for AccuracyMetric { + fn from(name: MetricName) -> Self { + match name { + MetricName::Absolute => Self::ABSOLUTE, + MetricName::AbsoluteStreamL1 => Self::ABSOLUTE_STREAM_L1, + MetricName::AbsoluteStreamL2 => Self::ABSOLUTE_STREAM_L2, + MetricName::Relative => Self::Relative, + MetricName::Rank => Self::Rank, + } + } +} + +/// Which output values one (ε, δ) statement covers *simultaneously*. +/// +/// A scope bounds only the failure probability: a per-value guarantee with +/// failure probability `δ` holds over the `n` values a scope covers with +/// `n · δ` by the union bound ([`Self::max_values_covered`]), whatever their +/// dependence. Not `Eq`: a record share is an `f64`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AccuracyScope { + /// Each output value (one series or row, at one evaluation) is within ε + /// of its true value with probability ≥ 1 − δ, separately. Values may + /// fail in different runs; nothing is said about all of them at once. + /// Sketches and composition produce guarantees of this scope. + PerValue, + /// Every value in one evaluation's returned result is within ε + /// simultaneously, with probability ≥ 1 − δ. The default of a target: a + /// target at a query root applies to the entire returned result. + WholeResult, + /// Every group (result row) holding at least this share of the records + /// the query aggregates in one evaluation is within ε simultaneously, + /// with probability ≥ 1 − δ; smaller groups are not covered. The share + /// is a fraction in (0, 1] (0.05 for 5%). When a row reads several + /// aggregations, its group must hold the share of each one's records. + /// At most `⌊1 / share⌋` groups can qualify, whatever the number of + /// groups. JSON: `{"groups_with_record_share_at_least": 0.05}`. + GroupsWithRecordShareAtLeast(#[serde(deserialize_with = "record_share")] f64), +} + +/// Relative tolerance for `⌊1 / share⌋`, so a share such as 0.05 whose +/// reciprocal computes just below an integer counts that integer. +const RECORD_SHARE_ROUNDING_TOLERANCE: f64 = 1e-9; + +/// A record share, rejected unless it is a fraction in (0, 1]. +fn record_share<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let share = f64::deserialize(deserializer)?; + if share > 0.0 && share <= 1.0 { + Ok(share) + } else { + Err(serde::de::Error::custom(format!( + "a record share is a fraction in (0, 1], got {share}" + ))) + } +} + +impl AccuracyScope { + /// How many values one evaluation's statement over this scope covers at + /// most, given the `values_per_result` (`G`) the result returns, `None` + /// when unbound: one for [`Self::PerValue`], `G` for + /// [`Self::WholeResult`], and `min(⌊1 / share⌋, G)` for + /// [`Self::GroupsWithRecordShareAtLeast`], which is bound even when `G` + /// is not. Sizing divides a target's failure probability by it, and the + /// guarantee check multiplies a per-value one by it, so the strictest of + /// several scopes is the one that covers the most values. + pub fn max_values_covered(self, values_per_result: Option) -> Option { + match self { + Self::PerValue => Some(1), + Self::WholeResult => values_per_result, + // A share outside (0, 1] (possible only through the Rust API; + // JSON rejects it) bounds nothing, so its target stays unmet. + Self::GroupsWithRecordShareAtLeast(share) if !(share > 0.0 && share <= 1.0) => None, + Self::GroupsWithRecordShareAtLeast(share) => { + // More than `1 / share` groups cannot each hold `share` of + // the records. Rounding up near an integer keeps `1 / 0.05` + // at 20 when it computes just below; it only enlarges the + // count, which is conservative. + let qualifying = + (share.recip() * (1.0 + RECORD_SHARE_ROUNDING_TOLERANCE)).floor() as u64; + Some(values_per_result.map_or(qualifying, |values| values.min(qualifying))) + } + } + } +} + +/// The outer shape of an accuracy guarantee +/// ([`crate::accuracy::AccuracyGuarantee`]): +/// `Pr[err_metric(estimate, truth) > error] ≤ failure_probability` over the +/// values `accuracy_scope` names. An [`AccuracyTarget`] has the same fields, with +/// concrete numbers, an optional metric and `max_error` for `error`: a target +/// states the largest error a query accepts, a guarantee the error bound +/// (an expression over the plan's parameters) its plan achieves. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AccuracySpec { + pub accuracy_metric: AccuracyMetric, + /// ε: the error bound, in `accuracy_metric`'s units. + pub error: E, + /// δ. + pub failure_probability: P, + pub accuracy_scope: AccuracyScope, +} + +/// An accuracy requirement, as a workload, a caller or a frontend states +/// it: `Pr[err_metric(estimate, truth) > max_error] ≤ failure_probability` over +/// the values `accuracy_scope` names. +/// +/// `accuracy_metric: None` means the user did not set one: plan selection +/// then accepts a guarantee in any metric whose ε, δ and scope meet this +/// target's, and reports the metric it accepted. A set metric accepts only +/// guarantees in that metric (or convertible to it with evidence). +/// `failure_probability: None` imposes no δ requirement (sizing then uses +/// the planner's default δ). Exact is `max_error = 0`, +/// `failure_probability = Some(0)`, whatever the metric. +/// +/// Built with [`Self::exact`] or [`Self::max_error`] plus the optional +/// builders [`Self::failure_probability`], [`Self::accuracy_metric`] and +/// [`Self::accuracy_scope`]. +/// +/// JSON: `{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": +/// 0.001, "accuracy_scope": "whole_result"}`; every field but `max_error` may be +/// omitted (`accuracy_scope` defaults to `whole_result`). Exact is +/// `{"max_error": 0.0, "failure_probability": 0.0}`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AccuracyTarget { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accuracy_metric: Option, + /// ε: the largest error the query accepts, in the metric's units. + pub max_error: f64, + /// δ. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_probability: Option, + #[serde(default = "whole_result")] + pub accuracy_scope: AccuracyScope, +} + +fn whole_result() -> AccuracyScope { + AccuracyScope::WholeResult +} + +impl AccuracyTarget { + /// No approximation permitted. + pub const fn exact() -> Self { + Self { + accuracy_metric: None, + max_error: 0.0, + failure_probability: Some(0.0), + accuracy_scope: AccuracyScope::WholeResult, + } + } + + /// Error at most `max_error` over the whole result, with no δ + /// requirement and no metric set. + pub const fn max_error(max_error: f64) -> Self { + Self { + accuracy_metric: None, + max_error, + failure_probability: None, + accuracy_scope: AccuracyScope::WholeResult, + } + } + + /// This target, met with probability at least `1 − failure_probability`. + pub const fn failure_probability(self, failure_probability: f64) -> Self { + Self { + failure_probability: Some(failure_probability), + ..self + } + } + + /// This target with its error stated in `accuracy_metric`. + pub const fn accuracy_metric(self, accuracy_metric: AccuracyMetric) -> Self { + Self { + accuracy_metric: Some(accuracy_metric), + ..self + } + } + + /// This target over `accuracy_scope`. + pub const fn accuracy_scope(self, accuracy_scope: AccuracyScope) -> Self { + Self { + accuracy_scope, + ..self + } + } + + /// Whether this target permits no approximation. + pub fn is_exact(&self) -> bool { + self.max_error == 0.0 && self.failure_probability == Some(0.0) + } + + /// The metrics this target accepts an error in: its metric, every + /// metric when it names none. + pub fn metrics(&self) -> Vec { + AccuracyMetric::ALL + .into_iter() + .filter(|&metric| self.accuracy_metric.is_none_or(|named| named == metric)) + .collect() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs new file mode 100644 index 000000000..6590a0377 --- /dev/null +++ b/crates/types/src/accuracy/target/tests.rs @@ -0,0 +1,334 @@ +use super::super::default_metric::column_metric; +use super::*; +use crate::accuracy::default_metric_for_root; +use crate::ir::operator::{AggIntent, Reduction, Source}; +use crate::ir::schema::{DataType, Field, Schema}; +use crate::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; +use std::rc::Rc; + +fn node(op: NonASAPOp) -> Rc { + Rc::new(OperatorNode::new(Operator::NonASAP(op)).expect("fixture derives a schema")) +} + +fn aggregate(measures: Vec) -> QueryRoot { + let scan = node(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }); + QueryRoot::Operator(node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![]), + measures, + output_names: vec![], + filters: vec![], + having: None, + child: scan, + })) +} + +fn quantile() -> AggIntent { + AggIntent::Quantile { col: None, q: 0.99 } +} + +/// A target reads and writes as a plain struct: every field but +/// `max_error` may be omitted (no metric, no δ, the whole result), an unset +/// metric or δ writes nothing, and the old named forms (`"Exact"`, +/// `{"Epsilon": ε}`, `{"EpsilonDelta": …}`) and unknown fields are +/// rejected. +#[test] +fn accuracy_target_json_is_a_plain_struct() { + use serde_json::json; + let read = |value: serde_json::Value| serde_json::from_value::(value); + assert_eq!( + read(json!({"max_error": 0.01})).unwrap(), + AccuracyTarget::max_error(0.01) + ); + assert_eq!( + read(json!({"max_error": 0.0, "failure_probability": 0.0})).unwrap(), + AccuracyTarget::exact() + ); + let rank = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::Rank); + assert_eq!( + read(json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.001})) + .unwrap(), + rank + ); + assert_eq!( + serde_json::to_value(&rank).unwrap(), + json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.001, + "accuracy_scope": "whole_result"}) + ); + assert_eq!( + serde_json::to_value(AccuracyTarget::max_error(0.01)).unwrap(), + json!({"max_error": 0.01, "accuracy_scope": "whole_result"}) + ); + for old in [ + json!("Exact"), + json!({"Epsilon": 0.01}), + json!({"EpsilonDelta": {"epsilon": 0.01, "delta": 0.001}}), + json!({"max_error": 0.1, "typo": true}), + ] { + assert!(read(old.clone()).is_err(), "{old}"); + } +} + +/// Metrics have stable wire names. +#[test] +fn metric_wire_names() { + for (metric, name) in [ + (AccuracyMetric::ABSOLUTE, "absolute"), + (AccuracyMetric::ABSOLUTE_STREAM_L1, "absolute_stream_l1"), + (AccuracyMetric::ABSOLUTE_STREAM_L2, "absolute_stream_l2"), + (AccuracyMetric::Relative, "relative"), + (AccuracyMetric::Rank, "rank"), + ] { + assert_eq!(serde_json::to_value(metric).unwrap(), name); + assert_eq!( + serde_json::from_value::(name.into()).unwrap(), + metric + ); + } +} + +/// A per-entity count (`count_over_time`) keeps every label column and +/// replaces the sample value in place: the value column carries the +/// count's metric, and a label after it is absolute. +#[test] +fn a_per_entity_count_states_its_value_column_not_its_last_column() { + let scan = node(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, false), + ], + 0, + vec![], + ), + }); + let count = node(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child: scan, + }); + assert_eq!(count.schema.fields[1].name, "value"); + assert_eq!( + column_metric(&count, 1), + Some(AccuracyMetric::ABSOLUTE_STREAM_L1) + ); + assert_eq!(column_metric(&count, 2), Some(AccuracyMetric::ABSOLUTE)); +} + +/// A root's default metric follows its computation: counts and top-k +/// frequencies are L1-scaled absolute errors, distinct counts and L2 +/// frequency moments relative ones, anything else (a sum or an entropy +/// among them) absolute value errors. +#[test] +fn root_metric_defaults_follow_the_query_intent() { + let metric = |intent: AggIntent| default_metric_for_root(&aggregate(vec![intent])).unwrap(); + assert_eq!(metric(AggIntent::Count), AccuracyMetric::ABSOLUTE_STREAM_L1); + assert_eq!( + metric(AggIntent::Sum { col: None }), + AccuracyMetric::ABSOLUTE + ); + assert_eq!( + metric(AggIntent::TopK { k: 5 }), + AccuracyMetric::ABSOLUTE_STREAM_L1 + ); + assert_eq!( + metric(AggIntent::Cardinality { cols: vec![] }), + AccuracyMetric::Relative + ); + assert_eq!( + metric(AggIntent::FrequencyL2 { col: None }), + AccuracyMetric::Relative + ); + assert_eq!( + metric(AggIntent::FrequencyEntropy { col: None }), + AccuracyMetric::ABSOLUTE + ); + assert_eq!( + metric(AggIntent::Avg { col: None }), + AccuracyMetric::ABSOLUTE + ); + assert_eq!( + default_metric_for_root(&QueryRoot::Scalar(ScalarExpr::literal_f64(1.0))), + Some(AccuracyMetric::ABSOLUTE) + ); +} + +/// SQL's empty-input NULL guard (a statistic cross-joined with an exact +/// row count, and `CASE WHEN count = 0 THEN NULL ELSE statistic END`) +/// keeps the statistic's metric: the root's value is the statistic's +/// column. A `CASE` whose value branches read columns of different +/// metrics, or any other expression, is absolute. +#[test] +fn root_metric_looks_through_a_null_guard_to_its_statistic() { + use crate::ir::operator::arguments::JoinKind; + use crate::ir::scalar::{CompareOpKind, ScalarValue}; + use crate::ir::{Predicate, ProjectItem}; + let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { col: None }]) else { + unreachable!() + }; + let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count]) else { + unreachable!() + }; + let joined = node(NonASAPOp::Join { + kind: JoinKind::Cross, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: l2, + right: count, + }); + let guard = |otherwise: ScalarExpr| { + QueryRoot::Operator(node(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: crate::ir::ExprSemantics::Sql, + }, + ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Literal(ScalarValue::Null)), + to: DataType::Float64, + try_cast: false, + }, + )], + else_expr: Some(Box::new(otherwise)), + }, + }], + child: joined.clone(), + })) + }; + let metric = |root: QueryRoot| default_metric_for_root(&root).unwrap(); + assert_eq!( + metric(guard(ScalarExpr::Column(0))), + AccuracyMetric::Relative + ); + // The count's column, or arithmetic over the statistic. + assert_eq!( + metric(guard(ScalarExpr::Column(1))), + AccuracyMetric::ABSOLUTE_STREAM_L1 + ); + assert_eq!( + metric(guard(ScalarExpr::Negative { + expr: Box::new(ScalarExpr::Column(0)), + semantics: crate::ir::ExprSemantics::Sql, + })), + AccuracyMetric::ABSOLUTE + ); +} + +/// A quantile root has no default metric: its error is as naturally a +/// rank as a relative one, and a target that sets none accepts either. +/// It is not an input error. +#[test] +fn a_quantile_root_has_no_default_metric() { + assert_eq!(default_metric_for_root(&aggregate(vec![quantile()])), None); +} + +/// A target on the groups holding at least a share of the records reads +/// and writes its share as a fraction; a share outside (0, 1] (say 5 +/// meant as 5%, or 0, which would cover every group) is bad input. +#[test] +fn a_record_share_scope_reads_its_share_as_a_fraction() { + use serde_json::json; + let target = AccuracyTarget::max_error(0.05) + .failure_probability(0.05) + .accuracy_scope(AccuracyScope::GroupsWithRecordShareAtLeast(0.05)); + let wire = json!({"max_error": 0.05, "failure_probability": 0.05, + "accuracy_scope": {"groups_with_record_share_at_least": 0.05}}); + assert_eq!(serde_json::to_value(&target).unwrap(), wire); + assert_eq!( + serde_json::from_value::(wire).unwrap(), + target + ); + for share in [json!(0.0), json!(-0.1), json!(5.0), json!(1.5)] { + let bad = json!({"max_error": 0.05, + "accuracy_scope": {"groups_with_record_share_at_least": share}}); + assert!( + serde_json::from_value::(bad).is_err(), + "{share}" + ); + } +} + +/// How many values one statement covers at most: one for a per-value +/// scope, the `G` values of the result for the whole result (unbound when +/// `G` is), and at most `⌊1 / share⌋` groups for a record share, since +/// more groups cannot each hold that share. So a 1% scope covers 100, +/// a 5% scope 20, fewer when the result has fewer values. +#[test] +fn a_scope_covers_at_most_one_over_its_share_groups() { + use AccuracyScope::*; + assert_eq!(PerValue.max_values_covered(None), Some(1)); + assert_eq!(WholeResult.max_values_covered(None), None); + assert_eq!(WholeResult.max_values_covered(Some(625)), Some(625)); + assert_eq!( + GroupsWithRecordShareAtLeast(0.05).max_values_covered(None), + Some(20) + ); + assert_eq!( + GroupsWithRecordShareAtLeast(0.01).max_values_covered(None), + Some(100) + ); + assert_eq!( + GroupsWithRecordShareAtLeast(0.03).max_values_covered(None), + Some(33) + ); + assert_eq!( + GroupsWithRecordShareAtLeast(1.0).max_values_covered(None), + Some(1) + ); + for invalid in [0.0, -0.1, 1.5, f64::NAN] { + assert_eq!( + GroupsWithRecordShareAtLeast(invalid).max_values_covered(None), + None, + "{invalid}" + ); + } + assert_eq!( + GroupsWithRecordShareAtLeast(0.01).max_values_covered(Some(4)), + Some(4) + ); +} + +/// Exact means zero error with certainty, whatever the metric; an +/// ε of zero without a δ is not exact. +#[test] +fn exact_targets() { + assert!(AccuracyTarget::exact().is_exact()); + assert_eq!(AccuracyTarget::exact().accuracy_metric, None); + assert!(AccuracyTarget::exact() + .accuracy_metric(AccuracyMetric::Rank) + .is_exact()); + assert!(!AccuracyTarget::max_error(0.0).is_exact()); + assert_eq!( + AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .failure_probability, + Some(0.001) + ); + assert_eq!( + AccuracyTarget::max_error(0.01).accuracy_scope, + AccuracyScope::WholeResult + ); +} diff --git a/crates/types/src/deployment_capabilities.rs b/crates/types/src/deployment_capabilities.rs index 4c973a95a..35ee3aa02 100644 --- a/crates/types/src/deployment_capabilities.rs +++ b/crates/types/src/deployment_capabilities.rs @@ -1,4 +1,4 @@ -//! What the deployment's executor can run and keep (#509's third +//! What the deployment's executor can run and keep (the third //! deployment input, beside the cost and accuracy models). //! //! [`DeploymentCapabilities`] says three things: diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index 35d1be914..9b945d2cb 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -13,7 +13,6 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use crate::accuracy::AccuracyGuarantee; use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::query::QueryRoot; use crate::ir::schema::Schema; @@ -27,7 +26,6 @@ pub struct FlatNode { pub operator: Operator, pub(crate) result_kind: OperatorResultKind, pub(crate) schema: Schema, - pub(crate) guarantee: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -73,7 +71,6 @@ fn visit( operator, result_kind: node.result_kind, schema: node.schema.clone(), - guarantee: node.guarantee.clone(), }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id); diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index b8cf64393..67029668a 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -1,4 +1,4 @@ -//! The operator IR from #511: one operator DAG for every planning stage, +//! The operator IR: one operator DAG for every planning stage, //! arranged by IR section. pub mod operator; pub mod properties; diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 2449391bf..d17b2aab3 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -33,7 +33,7 @@ use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType}; /// `None` is the PromQL convention "the time-series sample value"; SQL /// `SUM(bytes), AVG(latency)` sets distinct `Some(_)`s so a multi-aggregate /// node binds each reducer to the right column, and `plan::bind` knows which -/// column to summarise over (issue #115). `Cardinality` and `PearsonCorr` read +/// column to summarise over. `Cardinality` and `PearsonCorr` read /// more than one column, so they carry their own lists; [`input_cols`] is the /// arity-agnostic accessor every consumer goes through. /// @@ -261,9 +261,9 @@ pub enum AggIntent { TsOfLastOverTime, /// A deployment-model-specific intent this core vocabulary doesn't - /// (and shouldn't) know the shape of. First step toward issue #131 - /// ("add an explicit registration API for extending query / sketch / - /// primitive / data types") applied to this IR: rather than growing + /// (and shouldn't) know the shape of. A first step toward an explicit + /// registration API for extending query / sketch / + /// primitive / data types, applied to this IR: rather than growing /// `AggIntent` for every capability a single deployment model needs /// (which would turn this shared vocabulary into a dumping ground — /// core only grows for intents ≥2 deployment models actually use), @@ -367,7 +367,7 @@ pub enum MathFunc { // only its presence via a `{ .. }` pattern — so, unlike schema derivation // (which needs bound positions), nothing stops these from being generic // over every `C`. And a front end constructing `AggIntent` -// directly (issue #179) does need `is_per_series` pre-binding — it decides +// directly does need `is_per_series` pre-binding — it decides // the `PerEntity`/`Reduce` reduction shape right at construction time (see // `asap_frontend_promql::promql::reduction_for`) — so they stay generic // alongside `input_col`, in one `impl` block. diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index ed557c4b6..58e3bc768 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -239,7 +239,7 @@ pub enum JoinKind { Full, Cross, /// Left semi-join — each left row that has **at least one** match, once. - /// `WHERE c IN (SELECT …)` / `WHERE EXISTS (…)` (issue #111). + /// `WHERE c IN (SELECT …)` / `WHERE EXISTS (…)`. /// /// Output schema is the **left's alone**; the right side is a filter, not a /// source of columns. The join predicate still resolves against the @@ -285,10 +285,10 @@ pub enum WindowFuncKind { /// these respect the window frame bounds (NULL/default past the frame edge) /// rather than reaching arbitrarily far back/forward. Kept as distinct /// variants so the frame clause is never silently discarded by conflating - /// them with `Lag`/`Lead` (#267). `WindowFuncKind` still has no frame + /// them with `Lag`/`Lead`. `WindowFuncKind` still has no frame /// representation, so today these lower and behave exactly like /// `Lag`/`Lead` — the tag is correct, the frame-respecting behavior isn't - /// implemented yet. See #231 for modeling window frames properly. + /// implemented yet; window frames need proper modeling. LagInFrame, LeadInFrame, FirstValue, @@ -304,7 +304,7 @@ pub enum WindowFuncKind { /// A window's frame-spec (`ROWS`/`RANGE BETWEEN … AND …`) — which rows around /// the current one an analytic window function reads. `GROUPS` is rejected at -/// lowering time (issue #268): every SQL corpus in this repo uses only `ROWS`, +/// lowering time: every SQL corpus in this repo uses only `ROWS`, /// and nothing downstream interprets frame semantics yet, so it isn't worth /// modelling untested. /// @@ -450,7 +450,7 @@ pub enum GroupSide { /// `Reduce`). Explicit, decided once by whichever pass constructs the node /// (structural, at front-end lowering time), rather than inferred downstream from /// whether a grouping-key list happens to be empty or from a neighboring -/// node's shape. See design proposal #165. +/// node's shape. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Reduction { /// Collapses input rows via `by` — `by`/`without` semantics are exactly diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index edac475a3..1b45b6fda 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -84,7 +84,7 @@ pub enum ASAPOp> { /// It has no range and no window count: a `TimeRange` (and /// `TimeShift`) over it selects the windows a query reads, and how many /// windows it retains is $h_v$, a Stage 3 choice. - // Written as `PaneChain` before #1163; the aliases read those documents. + // Written as `PaneChain` before the rename; the alias reads those documents. #[serde(alias = "PaneChain")] SummaryTumblingWindowChain { child: C, diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index b9b53b2a4..5da385dc3 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -1,4 +1,4 @@ -//! #511 §1: the operators of the unified DAG and the parameter types their +//! The operators of the unified DAG and the parameter types their //! payloads are built from. //! //! - [`node`] — [`OperatorNode`](node::OperatorNode): an operator plus the diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 4f4a0a685..19910108e 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -1,6 +1,6 @@ //! The unified operator node: one node type before and after ASAP //! optimization. A node is an operator plus the common planning properties -//! every traversal needs (output category and schema, accuracy guarantee). +//! every traversal needs (output category and schema). //! Physical properties (timing, lifetime, schedule) are not on the node: //! each physical candidate holds its own //! ([`crate::stages::PhysicalCandidate::properties`]). @@ -14,7 +14,6 @@ use super::arguments::Source; use super::asap::ASAPOp; use super::input::OperatorInput; use super::non_asap::NonASAPOp; -use crate::accuracy::AccuracyGuarantee; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -117,14 +116,12 @@ impl Operator { /// node. /// /// `schema` and `result_kind` are derived from `operator` and its children -/// at construction and retained. `guarantee` is `None` until accuracy -/// assessment establishes one (`None` never means exact). +/// at construction and retained. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, - pub guarantee: Option, } impl OperatorNode { @@ -145,7 +142,6 @@ impl OperatorNode { operator, result_kind, schema, - guarantee: None, } } @@ -156,11 +152,6 @@ impl OperatorNode { Self::new(operator).map(Rc::new) } - pub fn with_guarantee(mut self, guarantee: Option) -> Self { - self.guarantee = guarantee; - self - } - /// The build whose state this summary-state node carries /// ([`summary_build`](super::asap::summary_build)). pub fn summary_build(self: &Rc) -> Option { @@ -208,7 +199,6 @@ impl OperatorNode { /// structural schema metadata. Only names that override the old derived /// schema are retained, for either operator category. A change in output /// arity with such overrides needs an explicit new naming assignment. - /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, f: impl FnMut(&Rc) -> Rc, diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 065a3a310..96debd9e1 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -3,8 +3,8 @@ //! with a time index. A chain has no range and no window count: a //! `TimeRange` (and `TimeShift`s) over it select the windows a query reads, //! a `SummaryMerge` over that selection is the state of the chain's build -//! over the same range, and how many windows it retains is $h_v$ (#1207 -//! PR 7). +//! over the same range, and how many windows it retains is $h_v$ (not yet +//! specified). use super::asap::ASAPOp; use super::input::OperatorInput; use super::node::Operator; diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 46061a86d..af33f377b 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -1,4 +1,4 @@ -//! #511 §2.2–2.3: the planning properties a node carries besides its +//! The planning properties a node carries besides its //! operator and schema (when its value is produced), and the selection //! algebra aggregate coverage is stated in. Aggregate coverage itself is a //! memo-group property, derived and stored by the memo (`crate::stages`). diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 06b7cc02c..3db26cf21 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -167,7 +167,7 @@ mod limit_bound { /// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. -pub(crate) fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { +pub fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { let literal = |e: &ScalarExpr| match e { ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), _ => None, @@ -443,8 +443,8 @@ impl Constraint { } } -#[cfg(any(test, feature = "test-support"))] impl SelectionBox { + #[cfg(any(test, feature = "test-support"))] pub fn new(columns: BTreeMap) -> Self { Self { columns } } @@ -453,9 +453,7 @@ impl SelectionBox { pub fn columns(&self) -> &BTreeMap { &self.columns } -} -impl SelectionBox { /// Whether no row lies in both boxes: some column is restricted by both /// and the restrictions are disjoint. pub(crate) fn disjoint(&self, other: &Self) -> bool { diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index 5fe7edd11..f32702697 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -90,7 +90,7 @@ fn per_series_reduction_schema( /// `per_series_reduction_schema` (`rate`/`increase`/`*_over_time`) instead /// of the cross-series `by ++ measures` shape. Which one applies is read directly /// off `reduction` — decided once, at construction, by whoever built the -/// `Aggregate` node (issue #165) — not re-derived here from `by`/child shape. +/// `Aggregate` node — not re-derived here from `by`/child shape. pub fn aggregate_output_schema( in_schema: &Schema, reduction: &Reduction, diff --git a/crates/types/src/ir/schema/series_identity.rs b/crates/types/src/ir/schema/series_identity.rs index 61bc53e69..3edf058b7 100644 --- a/crates/types/src/ir/schema/series_identity.rs +++ b/crates/types/src/ir/schema/series_identity.rs @@ -90,9 +90,7 @@ pub fn with_promql_series_identity( ) => {} _ => return Err("operator has no dynamic series-identity realization".into()), } - let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; - rebuilt.guarantee = node.guarantee.clone(); - let rebuilt = Rc::new(rebuilt); + let rebuilt = Rc::new(OperatorNode::new(operator).map_err(|e| e.to_string())?); cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); Ok(rebuilt) } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index d805699f6..d3fb45383 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -1,7 +1,7 @@ //! `asap-types` — shared vocabulary for the whole workspace. Every planning //! stage depends on it. //! -//! - [`ir`] — the unified operator IR (#511): one operator DAG for logical +//! - [`ir`] — the unified operator IR: one operator DAG for logical //! and physical plans ([`ir::OperatorNode`]), arranged by IR section: //! [`ir::operator`] (§1 operators and their argument types), //! [`ir::scalar`] (§2.2 scalar expressions), [`ir::schema`] (§2.1 the @@ -15,7 +15,7 @@ //! - [`deployment_capabilities`] — what the deployment's executor can run and //! keep ([`deployment_capabilities::DeploymentCapabilities`]), the deployment //! input Stages 1–3 read. -//! - [`workload`] — the planner inputs (#509): the query and data +//! - [`workload`] — the planner inputs: the query and data //! workloads and the parsed workload the query-language frontends produce //! ([`workload::parsed_workload`]). //! - [`stages`] — the types one planning stage hands the next: the Stage 1 @@ -26,7 +26,7 @@ //! - [`plan`] — Stage 3's output: the physical ASAP DAG //! ([`plan::physical_asap_dag`]) and the materialized views it installs. //! -//! No execution logic lives in this crate (issue #190). +//! No execution logic lives in this crate. pub mod accuracy; pub mod deployment_capabilities; pub mod ir; diff --git a/crates/types/src/plan/mod.rs b/crates/types/src/plan/mod.rs index c5e16eae8..8729cbcf9 100644 --- a/crates/types/src/plan/mod.rs +++ b/crates/types/src/plan/mod.rs @@ -3,3 +3,6 @@ //! ([`materialized_view`]). pub mod materialized_view; pub mod physical_asap_dag; +mod view_coverage; + +pub use view_coverage::view_coverage; diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 72fa6f2c6..91a5c4243 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -6,7 +6,7 @@ //! directly as its last step, one node per selected execution, each with the //! timing and lifetime of the physical candidate it was built from. //! The deployment executes the plan as given -//! (#509 Assumption 8): it does not change lifetime or timing. +//! (the physical plan is final): it does not change lifetime or timing. use std::collections::{HashMap, HashSet}; #[cfg(any(test, feature = "test-support"))] @@ -23,7 +23,6 @@ use crate::ir::operator::node::OperatorNode; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::scalar::ScalarExpr; -use crate::accuracy::AccuracyGuarantee; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::selection::SelectionBox; use crate::ir::schema::Schema; @@ -72,7 +71,7 @@ pub struct PhysicalASAPDAGNode { pub output_state: ExecutionDataState, /// How long and where the deployment materializes this node's output: the /// lifetime the physical properties assignment chose for the sub-DAG this - /// node produces. Any node may be materialized (#509 §Materialization); a node + /// node produces. Any node may be materialized; a node /// with no assignment has lifetime `None`. With `output_state.timing` it /// is the node's `PhysicalProperties`. pub lifetime: Lifetime, @@ -82,7 +81,6 @@ pub struct PhysicalASAPDAGNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub parameters: Option, pub output_schema: Schema, - pub guarantee: Option, /// For a materialized view (a node whose lifetime materializes its output /// across evaluations), how far back from the newest evaluation /// time its output must stay available once refreshed; for a query-time @@ -119,7 +117,7 @@ pub struct ViewCoverage { /// coverage's `AggregateDefinition` with every input group expanded /// recursively to its expression, down to the `Scan`s, holding no /// memo group or plan node ids, canonically serialized and hashed - /// (`stages::view_coverage`). + /// (`plan::view_coverage`). /// Equal across plans and processes that compute the same thing, /// whatever the selection, and different when they do not (a rollup to /// a coarser grouping computes something else and has its own). Inside @@ -217,8 +215,7 @@ pub struct PhysicalASAPDAG { pub roots: Vec, /// One expression per standalone scalar query, keyed by the query's /// position in the batch, in increasing order. Its operator references - /// name nodes of this DAG. A standalone scalar query is not a node (#511 - /// §2.3), so query `i` of the batch is the scalar root keyed `i` when + /// name nodes of this DAG. A standalone scalar query is not a node, so query `i` of the batch is the scalar root keyed `i` when /// there is one, and otherwise the next entry of `roots`. pub scalar_roots: Vec<(usize, ScalarExpr)>, } @@ -303,9 +300,8 @@ impl PhysicalASAPDAG { .collect()) } - /// Every node of this DAG as IR, by node id, with its exported - /// guarantee (an operator node has no physical properties): what - /// [`Self::query_roots`] reads. Shared inputs stay shared. Validates the + /// Every node of this DAG as IR, by node id (an operator node has no + /// physical properties): what [`Self::query_roots`] reads. Shared inputs stay shared. Validates the /// DAG first. #[cfg(any(test, feature = "test-support"))] fn operator_nodes( @@ -331,8 +327,7 @@ impl PhysicalASAPDAG { let operator = node .payload .map_children(|child| Rc::clone(&rebuilt[child])); - let restored = OperatorNode::with_schema(operator, node.output_schema.clone()) - .with_guarantee(node.guarantee.clone()); + let restored = OperatorNode::with_schema(operator, node.output_schema.clone()); rebuilt.insert(id, Rc::new(restored)); } } @@ -652,7 +647,6 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( lifetime: Lifetime::None, parameters: None, output_schema: flat_node.schema, - guarantee: flat_node.guarantee, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/types/src/stages/logical/view_coverage.rs b/crates/types/src/plan/view_coverage.rs similarity index 94% rename from crates/types/src/stages/logical/view_coverage.rs rename to crates/types/src/plan/view_coverage.rs index 06b2bebc8..f640ff1fa 100644 --- a/crates/types/src/stages/logical/view_coverage.rs +++ b/crates/types/src/plan/view_coverage.rs @@ -1,6 +1,6 @@ //! A memo group's view coverage ([`view_coverage`]): the plan's content id //! for the group's aggregate coverage, which names no group. The memo -//! derives aggregate coverage ([`super::aggregate_coverage`]) over memo +//! derives aggregate coverage ([`AggregateCoverage`](crate::stages::AggregateCoverage)) over memo //! groups and stores it per group (`GroupProperties::aggregate_coverage`); //! this file expands the stored coverage's input groups and hashes the //! result. @@ -43,7 +43,7 @@ pub fn view_coverage(memo: &Memo, group: MemoGroupId) -> Option { } /// An aggregate definition -/// ([`AggregateDefinition`](super::AggregateDefinition)) with its input +/// ([`AggregateDefinition`](crate::stages::AggregateDefinition)) with its input /// groups expanded ([`expanded`]): the fields in the same order, so it /// serializes as the definition would with each group replaced. #[derive(Serialize)] diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs deleted file mode 100644 index fd561c7e6..000000000 --- a/crates/types/src/stages/logical/candidate.rs +++ /dev/null @@ -1,282 +0,0 @@ -//! Candidate scope: each group's candidates $v = (e_v, p^{\log}_v)$ -//! (concepts doc `memo-groups-properties-requirements.md`, section 3; -//! #1203). - -use crate::ir::operator::asap::summary_type_of_operator; -use crate::ir::schema::FieldDataType; -use crate::ir::Operator; -use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId}; -use crate::summary_type::{parameter_list, SketchParams}; - -/// One candidate: position `index_in_group` in group `group`'s candidates -/// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] -pub struct CandidateForMemoGroupId { - pub group: MemoGroupId, - pub index_in_group: usize, -} - -/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. -#[derive(Debug, Clone)] -pub struct CandidateForMemoGroup { - /// $e_v$. - pub expression: CandidateExpression, - /// $p^{\log}_v$. - pub logical_properties: LogicalProperties, - /// $r^{\log}_{c,i}$, one per input group: per child of - /// `expression.operator`, in [`Operator::children`] order. Changed only - /// through [`Memo::set_candidate_input_requirements`](crate::stages::Memo::set_candidate_input_requirements). - pub input_requirements: Vec, - /// A fused candidate computes, as one operator, a sub-DAG pattern of - /// several operators in one appearance of the sub-expression (a top-k - /// over an aggregate as one heap summary; a sum over a sum as one - /// aggregate): the groups of the operators it replaces. Empty otherwise. - /// Reuse across appearances is Pass 2's, not fusion (concepts doc §3, - /// "Fused groups"). - pub fused_groups: Vec, - /// Which rule added it. Not in the formulation: kept for - /// `stage_pipeline --text` and the viewer (approved 2026-11-05). - pub origin: OriginOfCandidateForMemoGroup, -} - -/// Which rule added a candidate. Nothing in planning reads it. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] -pub enum OriginOfCandidateForMemoGroup { - /// Pass 1: exact, exact accumulator, or a summary type. - Pass1, - /// Canonical value preprocessing feeding a summary build. - SummaryInput, - /// A tumbling-window chain, or a merge reading one, of a summary state group. - TumblingWindows, - /// A coarse group regrouping a finer group. - Rollup, - /// A top-k read from another group's ranking. - TopKReuse, - /// Its metric's scans widened to a shared label set, read through a - /// `Project` (scan widening). - WidenedScans, -} - -/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with -/// its arguments, whose children are groups. The input groups $G_{v,i}$ are -/// [`Operator::children`], in that order. -#[derive(Debug, Clone, PartialEq)] -pub struct CandidateExpression { - pub operator: Operator, -} - -/// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. -#[derive(Debug, thiserror::Error)] -pub enum CandidateError { - /// The candidate's expression is not valid over its input groups (its - /// output schema does not derive, an input fails validation, or a merge - /// reads states it may not merge): a bug in the rule that built it. - #[error("the candidate's expression is not valid over its input groups: {0}")] - NotValidOverInputGroups(MemoGroupError), - /// Its output schema or output kind differs from the group's: it does - /// not compute the group's result (a bug in the rule that built it). - #[error("the candidate's output differs from group {group:?}'s")] - OutputDiffersFromGroup { group: MemoGroupId }, -} - -/// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's -/// argument, not a property, and a tumbling-window chain's width is the -/// chain operator's argument. [`Self::new`] rejects parameters of another -/// algorithm, not in the algorithm's list, missing on a sketch build (a chain -/// included) or present on an exact operator. -#[derive(Debug, Clone)] -pub struct LogicalProperties { - /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for - /// an exact operator. - pub(super) parameters: Option, -} - -impl LogicalProperties { - pub fn new( - expression: &CandidateExpression, - parameters: Option, - ) -> Result { - let algorithm = match summary_type_of_operator(&expression.operator) { - Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), - _ => None, - }; - match (algorithm, ¶meters) { - (Some(_), None) => return Err(LogicalPropertiesError::SketchBuildWithoutParameters), - (None, Some(_)) => return Err(LogicalPropertiesError::ParametersOnExactOperator), - (Some(algorithm), Some(parameters)) => { - if !parameter_list(algorithm).contains(parameters) { - return Err(LogicalPropertiesError::ParametersNotInList); - } - } - (None, None) => {} - } - Ok(Self { parameters }) - } - - /// $\boldsymbol\theta_v$. - pub fn parameters(&self) -> Option<&SketchParams> { - self.parameters.as_ref() - } -} - -/// Logical properties that do not fit their operator: bad input to -/// [`LogicalProperties::new`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum LogicalPropertiesError { - #[error("an exact operator has no summary parameters")] - ParametersOnExactOperator, - #[error("a sketch build needs parameters from its algorithm's list")] - SketchBuildWithoutParameters, - #[error("the parameters are not in their algorithm's predefined list")] - ParametersNotInList, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::ASAPOp; - use crate::summary_type::SketchAlgorithm; - - use crate::ir::operator::arguments::Reduction; - - use crate::ir::scalar::expr_ir::ColumnRef; - - use crate::ir::schema::FieldDataType; - use crate::summary_type::{ExactKind, HydraParams, SummaryUpdate}; - - /// `SummaryAgg` over group 0 building a state of `summary_type`. - fn summary_agg(summary_type: FieldDataType) -> CandidateExpression { - CandidateExpression { - operator: Operator::ASAP(ASAPOp::SummaryAgg { - child: MemoGroupId(0), - summary_type, - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }), - } - } - - const KLL_269: SketchParams = SketchParams::Kll { k: 269 }; - - const CMS: SketchParams = SketchParams::Cms { - width: 272, - depth: 5, - }; - - const HYDRA_CMS: SketchParams = SketchParams::Hydra(HydraParams::HydraCms { - width: 272, - depth: 5, - shared_rows: 3, - shared_columns: 1_024, - }); - - /// `LogicalProperties::new` over every operator kind and parameters: a - /// sketch build (a chain included) needs parameters of its algorithm from - /// its list; parameters are rejected on an operator that builds no - /// sketch. - #[test] - fn logical_properties_new_over_every_operator_and_parameters() { - use LogicalPropertiesError as Invalid; - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); - let kll_chain = CandidateExpression { - operator: Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: MemoGroupId(0), - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - tumbling_window_width_ms: 5_000, - }), - }; - let readout = CandidateExpression { - operator: Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: MemoGroupId(0), - query: crate::summary_type::SketchStatistic::Quantile { q: 0.5 }, - }), - }; - // k = 270 is KLL's structure but not in its list. - let parameters = [ - None, - Some(KLL_269), - Some(CMS), - Some(HYDRA_CMS), - Some(SketchParams::Kll { k: 270 }), - ]; - let no_parameters = Err(Invalid::ParametersOnExactOperator); - let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); - let not_in_list = Err(Invalid::ParametersNotInList); - let exact = [ - Ok(()), - no_parameters, - no_parameters, - no_parameters, - no_parameters, - ]; - // Another algorithm's parameters are not in KLL's list either. - let kll_build = [ - needs_parameters, - Ok(()), - not_in_list, - not_in_list, - not_in_list, - ]; - // Per operator: the result for each of `parameters`. - let table = [ - (summary_agg(kll), kll_build), - // No Hydra θ is listed yet, so no Hydra build is valid. - ( - summary_agg(FieldDataType::Sketch(SketchAlgorithm::HydraCms)), - [ - needs_parameters, - not_in_list, - not_in_list, - not_in_list, - not_in_list, - ], - ), - (summary_agg(FieldDataType::exact(ExactKind::Sum)), exact), - (kll_chain, kll_build), - (readout, exact), - ]; - for (expression, by_parameters) in table { - for (parameters, expected) in parameters.iter().zip(by_parameters) { - let properties = LogicalProperties::new(&expression, parameters.clone()); - assert_eq!( - properties.as_ref().map(|_| ()).map_err(|e| *e), - expected, - "{:?} {parameters:?}", - expression.operator.kind_name() - ); - if let Ok(properties) = properties { - assert_eq!(properties.parameters(), parameters.as_ref()); - } - } - } - } - - /// `summary_type_of_operator` reads the summary type of a `SummaryAgg` - /// and of a chain, and of no other operator. - #[test] - fn summary_type_of_operator_reads_the_builds_summary_type() { - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); - let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: MemoGroupId(0), - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - tumbling_window_width_ms: 5_000, - }); - let merge = Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![MemoGroupId(0)], - group_by: None, - }); - assert_eq!( - summary_type_of_operator(&summary_agg(kll.clone()).operator), - Some(&kll) - ); - assert_eq!(summary_type_of_operator(&chain), Some(&kll)); - assert_eq!(summary_type_of_operator(&merge), None); - } -} diff --git a/crates/types/src/stages/logical/candidate/mod.rs b/crates/types/src/stages/logical/candidate/mod.rs new file mode 100644 index 000000000..fae0c4d41 --- /dev/null +++ b/crates/types/src/stages/logical/candidate/mod.rs @@ -0,0 +1,167 @@ +//! Candidate scope: each group's candidates $v = (e_v, p^{\log}_v)$ +//! (concepts doc `memo-groups-properties-requirements.md`, section 3). + +use crate::ir::operator::asap::summary_type_of_operator; +use crate::ir::schema::FieldDataType; +use crate::ir::Operator; +use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId}; +use crate::summary_type::{parameter_list, SketchAlgorithm, SketchParams}; + +/// One candidate: position `index_in_group` in group `group`'s candidates +/// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] +pub struct CandidateForMemoGroupId { + pub group: MemoGroupId, + pub index_in_group: usize, +} + +/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. +#[derive(Debug, Clone)] +pub struct CandidateForMemoGroup { + /// $e_v$. + pub expression: CandidateExpression, + /// $p^{\log}_v$. + pub logical_properties: LogicalProperties, + /// $r^{\log}_{c,i}$, one per input group: per child of + /// `expression.operator`, in [`Operator::children`] order. Changed only + /// through [`Memo::set_candidate_input_requirements`](crate::stages::Memo::set_candidate_input_requirements). + pub input_requirements: Vec, + /// A fused candidate computes, as one operator, a sub-DAG pattern of + /// several operators in one appearance of the sub-expression (a top-k + /// over an aggregate as one heap summary; a sum over a sum as one + /// aggregate): the groups of the operators it replaces. Empty otherwise. + /// Reuse across appearances is Pass 2's, not fusion (concepts doc §3, + /// "Fused groups"). + pub fused_groups: Vec, + /// Which rule added it. Not in the formulation: kept for + /// `stage_pipeline --text` and the viewer. + pub origin: OriginOfCandidateForMemoGroup, +} + +impl CandidateForMemoGroup { + /// The candidate computing `operator` with θ `parameters`, added by + /// rule `origin`, fusing `fused_groups`. Its inputs have no requirement + /// yet: Pass 2 sets each readout's + /// (`Memo::set_candidate_input_requirements`). + pub fn new( + operator: Operator, + parameters: Option, + origin: OriginOfCandidateForMemoGroup, + fused_groups: Vec, + ) -> Self { + let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; + Self { + expression: CandidateExpression { operator }, + logical_properties: LogicalProperties { parameters }, + input_requirements, + fused_groups, + origin, + } + } +} + +/// The θ of each build candidate of `build`: every listed θ +/// (`parameter_list`) of a sketch build, `[None]` for any other operator +/// (an exact build). A group gets one build candidate per entry. +pub fn listed_build_parameters(build: &Operator) -> Vec> { + match summary_type_of_operator(build) { + Some(FieldDataType::Sketch(algorithm)) => parameter_list(*algorithm) + .iter() + .cloned() + .map(Some) + .collect(), + _ => vec![None], + } +} + +/// Which rule added a candidate. Nothing in planning reads it. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub enum OriginOfCandidateForMemoGroup { + /// Pass 1: exact, exact accumulator, or a summary type. + Pass1, + /// A tumbling-window chain, or a merge reading one, of a summary state group. + TumblingWindows, + /// A coarse group rolled up from a finer one, the compensating `Filter` + /// it reads, or a build of a finest common group. + Rollup, + /// A top-k read from another group's ranking. + TopKReuse, +} + +/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with +/// its arguments, whose children are groups. The input groups $G_{v,i}$ are +/// [`Operator::children`], in that order. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateExpression { + pub operator: Operator, +} + +/// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. +#[derive(Debug, thiserror::Error)] +pub enum CandidateError { + /// The candidate's expression is not valid over its input groups (its + /// output schema does not derive, an input fails validation, or a merge + /// reads states it may not merge): a bug in the rule that built it. + #[error("the candidate's expression is not valid over its input groups: {0}")] + NotValidOverInputGroups(MemoGroupError), + /// Its output schema or output kind differs from the group's: it does + /// not compute the group's result (a bug in the rule that built it). + #[error("the candidate's output differs from group {group:?}'s")] + OutputDiffersFromGroup { group: MemoGroupId }, + /// Its θ does not fit the group's output (a bug in the rule that built + /// it). + #[error("the candidate's parameters do not fit its group's output: {0}")] + LogicalProperties(LogicalPropertiesError), +} + +/// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's +/// argument, not a property, and a tumbling-window chain's width is the +/// chain operator's argument. A candidate passing states on without building +/// them (`SummaryMerge`, and a `Filter`, `TimeRange` or `TimeShift` over +/// states) carries the θ of the states it reads. +/// [`Memo::add_candidate`](crate::stages::Memo::add_candidate) checks θ +/// against the candidate's group's output ([`parameters_fit`]). +#[derive(Debug, Clone)] +pub struct LogicalProperties { + /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for + /// an exact operator. + pub(super) parameters: Option, +} + +impl LogicalProperties { + /// $\boldsymbol\theta_v$. + pub fn parameters(&self) -> Option<&SketchParams> { + self.parameters.as_ref() + } +} + +/// Whether θ `parameters` fit an output of sketch `algorithm` (`None`: no +/// sketch state): from its list, and present exactly for a sketch. +pub(super) fn parameters_fit( + algorithm: Option, + parameters: Option<&SketchParams>, +) -> Result<(), LogicalPropertiesError> { + match (algorithm, parameters) { + (Some(_), None) => Err(LogicalPropertiesError::SketchBuildWithoutParameters), + (None, Some(_)) => Err(LogicalPropertiesError::ParametersOnExactOperator), + (Some(algorithm), Some(parameters)) if !parameter_list(algorithm).contains(parameters) => { + Err(LogicalPropertiesError::ParametersNotInList) + } + _ => Ok(()), + } +} + +/// θ that does not fit the output of a candidate's group, as +/// [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum LogicalPropertiesError { + #[error("an exact operator has no summary parameters")] + ParametersOnExactOperator, + #[error("a sketch build, or a candidate passing sketch states on, needs parameters from its algorithm's list")] + SketchBuildWithoutParameters, + #[error("the parameters are not in their algorithm's predefined list")] + ParametersNotInList, +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/logical/candidate/tests.rs b/crates/types/src/stages/logical/candidate/tests.rs new file mode 100644 index 000000000..828f85a8d --- /dev/null +++ b/crates/types/src/stages/logical/candidate/tests.rs @@ -0,0 +1,50 @@ +use super::*; +use crate::ir::operator::arguments::Reduction; +use crate::ir::scalar::expr_ir::ColumnRef; +use crate::ir::ASAPOp; +use crate::summary_type::{ExactKind, SummaryUpdate}; +use crate::test_support::memo_fixtures::state; + +/// `summary_type_of_operator` reads the summary type of a `SummaryAgg` +/// and of a chain, and of no other operator. +#[test] +fn summary_type_of_operator_reads_the_builds_summary_type() { + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![MemoGroupId(0)], + group_by: None, + }); + assert_eq!( + summary_type_of_operator(&state(kll.clone(), MemoGroupId(0))), + Some(&kll) + ); + assert_eq!(summary_type_of_operator(&chain), Some(&kll)); + assert_eq!(summary_type_of_operator(&merge), None); +} + +/// A sketch build has one entry per listed θ, an exact build one +/// without θ. +#[test] +fn listed_build_parameters_lists_a_sketch_builds_parameters() { + let kll = SketchAlgorithm::Kll; + assert_eq!( + listed_build_parameters(&state(FieldDataType::Sketch(kll), MemoGroupId(0))), + parameter_list(kll) + .iter() + .cloned() + .map(Some) + .collect::>() + ); + assert_eq!( + listed_build_parameters(&state(FieldDataType::exact(ExactKind::Sum), MemoGroupId(0))), + vec![None] + ); +} diff --git a/crates/types/src/stages/logical/edge.rs b/crates/types/src/stages/logical/edge.rs index 4c9896618..9ccb9f809 100644 --- a/crates/types/src/stages/logical/edge.rs +++ b/crates/types/src/stages/logical/edge.rs @@ -1,4 +1,4 @@ -//! Edge scope: the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201). +//! Edge scope: the logical edges $(u, c, i)$ with $\alpha^{\log}$. use crate::stages::CandidateForMemoGroupId; diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/group/aggregate_coverage.rs similarity index 99% rename from crates/types/src/stages/logical/aggregate_coverage.rs rename to crates/types/src/stages/logical/group/aggregate_coverage.rs index a14e9552b..baef20aec 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/group/aggregate_coverage.rs @@ -26,7 +26,7 @@ use std::ops::Bound; use thiserror::Error; use super::column_properties::{project_lineage, ColumnLineage}; -use super::group::MemoGroupAsInput; +use super::MemoGroupAsInput; use crate::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; @@ -71,7 +71,6 @@ pub struct AggregateCoverage { pub(crate) selection: Vec, } -#[cfg(any(test, feature = "test-support"))] impl AggregateCoverage { pub fn definition(&self) -> &AggregateDefinition { &self.definition diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/group/column_properties.rs similarity index 99% rename from crates/types/src/stages/logical/column_properties.rs rename to crates/types/src/stages/logical/group/column_properties.rs index fe500ae94..82be7197e 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/group/column_properties.rs @@ -10,7 +10,7 @@ use std::collections::BTreeMap; -use super::group::{MemoGroupAsInput, MemoGroupId}; +use super::{MemoGroupAsInput, MemoGroupId}; use crate::ir::operator::arguments::{ BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, Source, SourceColumn, }; diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group/mod.rs similarity index 91% rename from crates/types/src/stages/logical/group.rs rename to crates/types/src/stages/logical/group/mod.rs index 44fa2f849..12b5a9bf9 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -3,18 +3,26 @@ use std::collections::BTreeSet; -use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage, CoverageError}; -use super::column_properties::{column_lineage_and_signs, ColumnLineage, ValueSign}; +mod aggregate_coverage; +mod column_properties; + +pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; +pub use column_properties::{ColumnLineage, ValueSign}; + +use aggregate_coverage::aggregate_coverage; +use column_properties::column_lineage_and_signs; + use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::OperatorInput; use crate::ir::scalar::column_resolution::ResolveError; use crate::ir::schema::topk::ranked_column; -use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::schema::{ColumnId, FieldDataType, Schema}; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; use crate::ir::{NonASAPOp, ScalarExpr}; use crate::ir::{Operator, OperatorResultKind}; use crate::stages::{CandidateForMemoGroup, Memo}; +use crate::summary_type::SketchAlgorithm; /// A group, by index in the memo's groups. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -92,6 +100,23 @@ pub struct GroupProperties { pub scanned_sources: BTreeSet, } +impl GroupProperties { + /// The sketch algorithm of the state the group outputs; `None` for + /// rows, values and exact states. + pub fn sketch_algorithm_of_state(&self) -> Option { + if self.result_kind != OperatorResultKind::State { + return None; + } + self.schema + .fields + .iter() + .find_map(|field| match field.dtype { + FieldDataType::Sketch(algorithm) => Some(algorithm), + _ => None, + }) + } +} + /// Why an expression is not valid over its input groups: what /// [`Memo::find_or_add_group`] and, wrapped in /// [`CandidateError::NotValidOverInputGroups`](crate::stages::CandidateError::NotValidOverInputGroups), diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing/mod.rs similarity index 83% rename from crates/types/src/stages/logical/memo/hash_consing.rs rename to crates/types/src/stages/logical/memo/hash_consing/mod.rs index 61c489187..60d0b925e 100644 --- a/crates/types/src/stages/logical/memo/hash_consing.rs +++ b/crates/types/src/stages/logical/memo/hash_consing/mod.rs @@ -60,15 +60,19 @@ pub(super) fn same_value(left: &T, right: &T) - } /// Whether `left` and `right` are one candidate of a group: the same -/// expression and logical properties (parameters), each by [`same_value`]: -/// typed equality first, serialized only when typed-equal. -pub(super) fn same_expression_and_logical_properties( - left: &CandidateForMemoGroup, - right: &CandidateForMemoGroup, -) -> bool { - same_value(&left.expression.operator, &right.expression.operator) +/// input requirements, so two readouts of one state with different required +/// accuracy are two candidates (compared first: it is cheap and tells +/// readout copies apart), and the same expression and logical properties +/// (parameters), each by [`same_value`] (typed equality first, serialized +/// only when typed-equal). +pub(super) fn same_candidate(left: &CandidateForMemoGroup, right: &CandidateForMemoGroup) -> bool { + left.input_requirements == right.input_requirements + && same_value(&left.expression.operator, &right.expression.operator) && same_value( &left.logical_properties.parameters, &right.logical_properties.parameters, ) } + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/logical/memo/hash_consing/tests.rs b/crates/types/src/stages/logical/memo/hash_consing/tests.rs new file mode 100644 index 000000000..07bfe87fd --- /dev/null +++ b/crates/types/src/stages/logical/memo/hash_consing/tests.rs @@ -0,0 +1,385 @@ +//! Hash-consing: equal expressions are one memo group, and a duplicate +//! candidate is the candidate it duplicates, on the concepts doc's KLL example +//! (section 1: G1 the latency scan, G2 its KLL state). + +use super::super::tests::kll_k; +use super::super::*; +use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::{ProjectItem, SortKey}; +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; +use crate::summary_type::SketchAlgorithm; +use crate::test_support::memo_builders::try_candidate; +use crate::test_support::memo_fixtures::{aggregate_with, scan, state}; + +/// An unordered `LIMIT 10` over `input`, whose copies may return +/// different rows. +fn unordered_limit(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: input, + }) +} + +/// An equal expression over the same input groups is found, not added. +#[test] +fn find_or_add_group_returns_the_group_of_an_equal_expression() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + assert_eq!( + memo.find_or_add_group(scan("latency", &[], vec![])) + .unwrap(), + g1 + ); + assert_eq!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(), + g2 + ); + assert_ne!( + memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(), + g1 + ); + assert_eq!(memo.groups().len(), 3); + assert_eq!( + memo.groups()[g2.0 as usize].expression.children(), + vec![&g1] + ); +} + +/// The same operator over different input groups is a different group. +#[test] +fn find_or_add_group_separates_equal_operators_over_different_inputs() { + let mut memo = Memo::new(&DataWorkload::default()); + let latency = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let errors = memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(); + assert_ne!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), latency)) + .unwrap(), + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), errors)) + .unwrap() + ); +} + +/// An expression whose copies may return different rows (an unordered +/// `LIMIT`) gets a new group every time. +#[test] +fn find_or_add_group_never_returns_an_unshareable_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + assert_ne!( + memo.find_or_add_group(unordered_limit(g1)).unwrap(), + memo.find_or_add_group(unordered_limit(g1)).unwrap() + ); +} + +/// KLL builds with k = 269 and k = 547 are two candidates of one group +/// G2: θ is a logical property, not part of the expression. +#[test] +fn kll_builds_with_k_269_and_k_547_share_one_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let k269_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let k269 = try_candidate( + &mut memo, + k269_group, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(269), + ) + .unwrap(); + let k547_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let k547 = try_candidate( + &mut memo, + k547_group, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(547), + ) + .unwrap(); + assert_eq!(k269_group, k547_group); + assert_ne!(k269, k547); + assert_eq!(memo.groups()[k269_group.0 as usize].candidates.len(), 2); + assert_eq!( + memo.candidate(k547).logical_properties.parameters(), + kll_k(547).as_ref() + ); +} + +/// A candidate with the same expression and logical properties as one +/// in the group is not added again; its id is returned. +#[test] +fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let first = try_candidate( + &mut memo, + g2, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(269), + ) + .unwrap(); + let again = try_candidate( + &mut memo, + g2, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(269), + ) + .unwrap(); + assert_eq!(first, again); + assert_eq!( + first, + CandidateForMemoGroupId { + group: g2, + index_in_group: 0 + } + ); + let other_size = try_candidate( + &mut memo, + g2, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(547), + ); + assert_ne!(other_size.unwrap(), first); + assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); +} + +/// `Scan m`: `[ts, service, value, latency]`, no unique key. +fn service_scan() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +/// Whether `a` and `b`, each built into one memo through +/// `find_or_add_group`, are one group. +fn one_group( + a: impl FnOnce(&mut Memo) -> MemoGroupId, + b: impl FnOnce(&mut Memo) -> MemoGroupId, +) -> bool { + let mut memo = Memo::new(&DataWorkload::default()); + let a = a(&mut memo); + let b = b(&mut memo); + a == b +} + +/// `operator` over the group of [`service_scan`]. +fn over_scan( + operator: impl FnOnce(MemoGroupId) -> Operator, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + |memo| { + let scan = memo.find_or_add_group(service_scan()).unwrap(); + memo.find_or_add_group(operator(scan)).unwrap() + } +} + +/// `quantile(q, col) by (by)` over [`service_scan`], with `filter`. +fn quantile_agg( + by: Vec, + col: Option, + q: f64, + filter: Option>, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + aggregate_with( + Reduction::by(by), + vec![(AggIntent::Quantile { col, q }, filter)], + None, + child, + ) + }) +} + +/// Aggregates that differ in a column, a quantile or a filter are +/// different groups. +#[test] +fn aggregates_that_differ_in_an_argument_are_different_groups() { + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(3), 0.5, None) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let latency_above_1 = Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Sql, + }); + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(2), 0.5, Some(latency_above_1)) + )); +} + +/// Equal expressions without a unique key are one group when they return +/// the same rows each time: a keyless scan, a global aggregate, a +/// `without` aggregate. +#[test] +fn equal_keyless_expressions_that_return_the_same_rows_are_one_group() { + let table_scan = |memo: &mut Memo| { + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + })) + .unwrap() + }; + assert!(one_group(table_scan, table_scan)); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let without = || { + over_scan(|child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }) + }; + assert!(one_group(without(), without())); +} + +/// `DISTINCT` on every column returns the same rows each time and is one +/// group; a `Dedup` on some columns keeps an unspecified row per key. +#[test] +fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + over_scan(move |child| Operator::NonASAP(NonASAPOp::Dedup { cols, child })) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); +} + +/// Two unordered `LIMIT 10`s may return different rows: two groups, and +/// so are equal parents over them. +#[test] +fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group( + over_scan(unordered_limit), + over_scan(unordered_limit) + )); + let distinct_over_limit = |memo: &mut Memo| { + let limit = over_scan(unordered_limit)(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Dedup { + cols: vec![], + child: limit, + })) + .unwrap() + }; + assert!(!one_group(distinct_over_limit, distinct_over_limit)); +} + +/// A `LIMIT` over a `Sort` with the same partitions keeps the first rows +/// of a fixed order (one group); over a `Sort` partitioned differently, +/// which rows come first depends on input order (two groups). +#[test] +fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + move |memo: &mut Memo| { + let sorted = over_scan(|child| { + Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child, + }) + })(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: sorted, + })) + .unwrap() + } + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); +} + +/// A projection emitting the literal `value`. +fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Literal(ScalarValue::Float64(value)), + }], + child, + }) + }) +} + +/// Signed zeros and different non-finite literals are different groups; +/// JSON's `null` for non-finite floats never decides equality. +#[test] +fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 517fb9d32..044e7decb 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -1,28 +1,31 @@ //! Memo scope: the memo, Stage 0's and Stage 1's one type, and each query //! over its groups; the operations over the whole memo -//! (concepts doc `memo-groups-properties-requirements.md`, section 3; #1203). +//! (concepts doc `memo-groups-properties-requirements.md`, section 3). use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; mod hash_consing; +use super::candidate::parameters_fit; use super::group::{ derive_group_properties, validate_expression_over_input_groups, MemoGroupAsInput, MemoGroupError, }; use crate::accuracy::AccuracyTarget; +use crate::ir::operator::arguments::Reduction; use crate::ir::operator::input::InputOrTemporaryBuild; use crate::ir::schema::Schema; -use crate::ir::Operator; use crate::ir::QueryRoot; +use crate::ir::{ASAPOp, Operator}; use crate::stages::{ - CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, - MemoGroupId, + CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, ColumnLineage, + LogicalRequirements, MemoGroup, MemoGroupId, }; -use crate::workload::{DataWorkload, SourceFacts}; +use crate::summary_type::SketchStatistic; +use crate::workload::bounds::source_name; +use crate::workload::{series_facts, DataWorkload, SourceFacts}; use hash_consing::{ - expression_hash, same_expression_and_logical_properties, same_serialized_expression, - serialized_expression, + expression_hash, same_candidate, same_serialized_expression, serialized_expression, }; /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with @@ -77,6 +80,86 @@ impl Memo { &self.groups } + /// The most values one evaluation of `group` returns, from the facts + /// the data workload declares; `None` when they bound it in no way. + /// The smaller of: + /// - the series (PromQL) or groups of rows (SQL) of the sources scanned + /// below it (`GroupProperties::scanned_sources`, each source's + /// `SourceFacts::series_count`, when every one declares it): every + /// value of a per-series or grouped result over them comes from at + /// least one of them; + /// - for a readout (`SummaryEstimate`, not a top-k) of a state, or a + /// state, grouped `by` keys, the groups those keys form: 1 for no + /// key, else the distinct combinations the source declares for keys + /// copied from one source (`SourceFacts::distinct_values_per_label` + /// and `distinct_values_per_label_set`). + /// + /// Examples: a readout of `quantile_over_time(0.9, m[5m])` over `m` + /// with 1000 series returns at most 1000 values, which `max(...)` over + /// it reads at once; `SELECT count(DISTINCT dst) FROM flows` reads one + /// distinct count per evaluation. + pub fn max_values_per_evaluation(&self, group: MemoGroupId) -> Option { + let series_scanned = self.groups[group.0 as usize] + .group_properties + .scanned_sources + .iter() + .try_fold(0u64, |total, source| { + let (facts, _) = series_facts(&self.source_facts, source_name(source))?; + total.checked_add(facts.series_count?) + }); + match (series_scanned, self.declared_groups_of_state(group)) { + (Some(series), Some(groups)) => Some(series.min(groups)), + (series, groups) => series.or(groups), + } + } + + /// The groups the `by` keys of the state `group` is, or reads, form + /// ([`Self::max_values_per_evaluation`]). + fn declared_groups_of_state(&self, group: MemoGroupId) -> Option { + match &self.groups[group.0 as usize].expression { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query, + }) if !matches!(query, SketchStatistic::TopK { .. }) => { + self.declared_groups_of_state(*summary_input) + } + Operator::ASAP( + ASAPOp::SummaryAgg { + reduction: Reduction::Reduce(by), + child, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + reduction: Reduction::Reduce(by), + child, + .. + }, + ) if !by.is_without() => { + let lineage = &self.groups[child.0 as usize] + .group_properties + .column_lineage; + let mut source = None; + let mut labels = Vec::new(); + for key in by.keys() { + let Some(ColumnLineage::SourceColumn(column)) = lineage.get(*key) else { + return None; + }; + if source.is_some_and(|source| source != &column.source) { + return None; + } + source = Some(&column.source); + labels.push(column.column.as_str()); + } + let Some(source) = source else { + return Some(1); + }; + let (facts, _) = series_facts(&self.source_facts, source_name(source))?; + facts.declared_distinct_combinations(&labels) + } + _ => None, + } + } + /// The queries, by index in the workload. pub fn queries(&self) -> &[MemoQuery] { &self.queries @@ -198,16 +281,19 @@ impl Memo { /// Adds `candidate` to `group`'s candidates and returns its id, unless /// the group already has a candidate with the same expression - /// (`same_expression`) and logical properties: then that candidate's - /// id, and `candidate` is dropped. A new candidate makes `group` a parent - /// of each of its input groups, and `group`'s queries reach each input - /// group and every group below it. + /// (`same_expression`), logical properties and input requirements: then + /// that candidate's id, and `candidate` is dropped. A new candidate makes + /// `group` a parent of each of its input groups, and `group`'s queries + /// reach each input group and every group below it. /// /// Fails, leaving the memo unchanged, when the candidate's expression /// is not valid over its input groups /// (`validate_expression_over_input_groups`), - /// or when its output schema or kind differs from the group's: then it - /// does not compute the group's result. + /// when its output schema or kind differs from the group's (then it + /// does not compute the group's result), or when its θ does not fit the + /// group's output: a candidate of a sketch state group (a build, or a + /// merge, `Filter`, `TimeRange` or `TimeShift` passing states on) has a + /// θ from that algorithm's list, any other candidate none. pub fn add_candidate( &mut self, group: MemoGroupId, @@ -216,7 +302,7 @@ impl Memo { if let Some(index_in_group) = self.groups[group.0 as usize] .candidates .iter() - .position(|known| same_expression_and_logical_properties(known, &candidate)) + .position(|known| same_candidate(known, &candidate)) { return Ok(CandidateForMemoGroupId { group, @@ -237,6 +323,11 @@ impl Memo { if schema != group_properties.schema || result_kind != group_properties.result_kind { return Err(CandidateError::OutputDiffersFromGroup { group }); } + parameters_fit( + group_properties.sketch_algorithm_of_state(), + candidate.logical_properties.parameters(), + ) + .map_err(CandidateError::LogicalProperties)?; let candidates = &mut self.groups[group.0 as usize].candidates; let input_groups: Vec = candidate .expression @@ -314,6 +405,34 @@ impl Memo { candidate.input_requirements = input_requirements; } + /// The candidates `consumer` may read on input `input`: those of the + /// input group, except that a consumer with a θ (a build, or a candidate + /// passing states on) reads only producers with the same θ or none. + pub fn producers( + &self, + consumer: CandidateForMemoGroupId, + input: usize, + ) -> Vec { + let candidate = self.candidate(consumer); + let group = *candidate.expression.operator.children()[input]; + let theta = candidate.logical_properties.parameters(); + self.groups[group.0 as usize] + .candidates + .iter() + .enumerate() + .filter( + |(_, producer)| match (theta, producer.logical_properties.parameters()) { + (Some(theta), Some(produced)) => theta == produced, + _ => true, + }, + ) + .map(|(index_in_group, _)| CandidateForMemoGroupId { + group, + index_in_group, + }) + .collect() + } + /// The groups, each after every group it or one of its candidates /// reads, the lower index first among those ready: a dependency order for /// candidate traversal, whatever order rules appended groups in. `None` @@ -355,16 +474,7 @@ impl Memo { } (order.len() == count).then_some(order) } -} -/// Lists `parent` among `group`'s parents, once. -fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { - if !group.parents.contains(&parent) { - group.parents.push(parent); - } -} - -impl Memo { /// The groups some query's root reads, ascending: the groups of the /// queries' roots that are a query's result or a scalar query's operand. pub fn root_groups(&self) -> BTreeSet { @@ -383,5 +493,12 @@ impl Memo { } } +/// Lists `parent` among `group`'s parents, once. +fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { + if !group.parents.contains(&parent) { + group.parents.push(parent); + } +} + #[cfg(test)] mod tests; diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 136f71d57..cd39aa94a 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -1,29 +1,17 @@ -//! The `Memo` methods: hash-consing groups ([`Memo::find_or_add_group`]) and -//! storing candidates ([`Memo::add_candidate`]), on the concepts doc's KLL +//! The `Memo` operations other than the equality they rely on: storing +//! candidates ([`Memo::add_candidate`]), parents and queries, requirements, +//! and output schemas of pending expressions, on the concepts doc's KLL //! example (section 1: G1 the latency scan, G2 its KLL state). use super::*; -use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; -use crate::ir::operator::AggIntent; -use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; -use crate::ir::scalar::{ProjectItem, SortKey}; -use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; -use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; +use crate::ir::operator::arguments::Reduction; +use crate::ir::schema::FieldDataType; +use crate::ir::NonASAPOp; use crate::stages::logical::group::MemoGroupAsInput; -use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; +use crate::stages::LogicalPropertiesError; use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; -use crate::test_support::memo_fixtures::{aggregate_with, readout, scan, state}; - -/// An unordered `LIMIT 10` over `input`, whose copies may return -/// different rows. -fn unordered_limit(input: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: input, - }) -} +use crate::test_support::memo_builders::{query, try_candidate}; +use crate::test_support::memo_fixtures::{readout, scan, state, state_grouped_by}; /// `Project(input)` keeping its first column. fn project_first_column(input: MemoGroupId) -> Operator { @@ -36,98 +24,11 @@ fn project_first_column(input: MemoGroupId) -> Operator { }) } -/// A candidate computing `operator` with `parameters`. -fn candidate( - operator: Operator, - parameters: Option, -) -> CandidateForMemoGroup { - let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; - CandidateForMemoGroup { - // Built directly: the doc example's k = 200 and k = 345 are not - // in today's KLL list, which `LogicalProperties::new` checks. - logical_properties: LogicalProperties { parameters }, - expression: CandidateExpression { operator }, - input_requirements, - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - } -} - -/// A query rooted at `group`, without an accuracy target or $n_q$. -fn query(group: MemoGroupId) -> MemoQuery { - MemoQuery { - root: QueryRoot::Operator(group), - accuracy_target: None, - number_of_values_per_query_evaluation: None, - } -} - /// KLL `k`. -fn kll_k(k: u32) -> Option { +pub(super) fn kll_k(k: u32) -> Option { Some(SketchParams::Kll { k }) } -/// An equal expression over the same input groups is found, not added. -#[test] -fn find_or_add_group_returns_the_group_of_an_equal_expression() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let g2 = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - assert_eq!( - memo.find_or_add_group(scan("latency", &[], vec![])) - .unwrap(), - g1 - ); - assert_eq!( - memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(), - g2 - ); - assert_ne!( - memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(), - g1 - ); - assert_eq!(memo.groups().len(), 3); - assert_eq!( - memo.groups()[g2.0 as usize].expression.children(), - vec![&g1] - ); -} - -/// The same operator over different input groups is a different group. -#[test] -fn find_or_add_group_separates_equal_operators_over_different_inputs() { - let mut memo = Memo::new(&DataWorkload::default()); - let latency = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let errors = memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(); - assert_ne!( - memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), latency)) - .unwrap(), - memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), errors)) - .unwrap() - ); -} - -/// An expression whose copies may return different rows (an unordered -/// `LIMIT`) gets a new group every time. -#[test] -fn find_or_add_group_never_returns_an_unshareable_group() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - assert_ne!( - memo.find_or_add_group(unordered_limit(g1)).unwrap(), - memo.find_or_add_group(unordered_limit(g1)).unwrap() - ); -} - /// A new group is a parent of each of its input groups, listed once. #[test] fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { @@ -144,92 +45,6 @@ fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); } -/// KLL builds with k = 200 and k = 345 are two candidates of one group -/// G2: θ is a logical property, not part of the expression. -#[test] -fn kll_builds_with_k_200_and_k_345_share_one_group() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let k200_group = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - let k200 = memo - .add_candidate( - k200_group, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(200), - ), - ) - .unwrap(); - let k345_group = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - let k345 = memo - .add_candidate( - k345_group, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(345), - ), - ) - .unwrap(); - assert_eq!(k200_group, k345_group); - assert_ne!(k200, k345); - assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); - assert_eq!( - memo.candidate(k345).logical_properties.parameters(), - kll_k(345).as_ref() - ); -} - -/// A candidate with the same expression and logical properties as one -/// in the group is not added again; its id is returned. -#[test] -fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let g2 = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - let first = memo - .add_candidate( - g2, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(200), - ), - ) - .unwrap(); - let again = memo - .add_candidate( - g2, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(200), - ), - ) - .unwrap(); - assert_eq!(first, again); - assert_eq!( - first, - CandidateForMemoGroupId { - group: g2, - index_in_group: 0 - } - ); - let other_size = candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(345), - ); - assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); - assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); -} - /// A new candidate makes its group a parent of each group it reads, and /// the group's queries reach those groups and every group below them. #[test] @@ -245,7 +60,7 @@ fn add_candidate_maintains_parents_and_queries() { let g3 = memo .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.95 }, g2)) .unwrap(); - assert_eq!(memo.add_query(query(g3)), 0); + assert_eq!(query(&mut memo, g3, None), 0); // A rule adds a state group G5 over another scan G4: no query // reaches them yet. let g4 = memo @@ -256,9 +71,11 @@ fn add_candidate_maintains_parents_and_queries() { .unwrap(); assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); // G3 gains a candidate reading G5: G5 and G4 now serve query 0. - memo.add_candidate( + try_candidate( + &mut memo, g3, - candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g5), None), + readout(SketchStatistic::Quantile { q: 0.95 }, g5), + None, ) .unwrap(); assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); @@ -268,7 +85,7 @@ fn add_candidate_maintains_parents_and_queries() { // Query 1 is G6 = Project(G3): it reaches G3 and everything below // it, through expressions and candidates, in ascending order. let g6 = memo.find_or_add_group(project_first_column(g3)).unwrap(); - assert_eq!(memo.add_query(query(g6)), 1); + assert_eq!(query(&mut memo, g6, None), 1); for group in [g1, g2, g3, g4, g5] { assert_eq!( memo.groups()[group.0 as usize].reached_by_queries(), @@ -292,7 +109,7 @@ fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { .unwrap(); let before = format!("{memo:?}"); assert!(matches!( - memo.add_candidate(g2, candidate(readout(SketchStatistic::Quantile { q: 0.5 }, g2), None)), + try_candidate(&mut memo, g2, readout(SketchStatistic::Quantile { q: 0.5 }, g2), None), Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 )); assert_eq!( @@ -329,9 +146,11 @@ fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { ); let before = format!("{memo:?}"); assert!(matches!( - memo.add_candidate( + try_candidate( + &mut memo, g3, - candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g1), None) + readout(SketchStatistic::Quantile { q: 0.95 }, g1), + None ), Err(CandidateError::NotValidOverInputGroups(_)) )); @@ -363,224 +182,6 @@ fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); } -/// `Scan m`: `[ts, service, value, latency]`, no unique key. -fn service_scan() -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - }) -} - -/// Whether `a` and `b`, each built into one memo through -/// `find_or_add_group`, are one group. -fn one_group( - a: impl FnOnce(&mut Memo) -> MemoGroupId, - b: impl FnOnce(&mut Memo) -> MemoGroupId, -) -> bool { - let mut memo = Memo::new(&DataWorkload::default()); - let a = a(&mut memo); - let b = b(&mut memo); - a == b -} - -/// `operator` over the group of [`service_scan`]. -fn over_scan( - operator: impl FnOnce(MemoGroupId) -> Operator, -) -> impl FnOnce(&mut Memo) -> MemoGroupId { - |memo| { - let scan = memo.find_or_add_group(service_scan()).unwrap(); - memo.find_or_add_group(operator(scan)).unwrap() - } -} - -/// `quantile(q, col) by (by)` over [`service_scan`], with `filter`. -fn quantile_agg( - by: Vec, - col: Option, - q: f64, - filter: Option>, -) -> impl FnOnce(&mut Memo) -> MemoGroupId { - over_scan(move |child| { - aggregate_with( - Reduction::by(by), - vec![(AggIntent::Quantile { col, q }, filter)], - None, - child, - ) - }) -} - -/// Aggregates that differ in a column, a quantile or a filter are -/// different groups. -#[test] -fn aggregates_that_differ_in_an_argument_are_different_groups() { - assert!(!one_group( - quantile_agg(vec![1], Some(2), 0.5, None), - quantile_agg(vec![1], Some(3), 0.5, None) - )); - assert!(!one_group( - quantile_agg(vec![], Some(2), 0.5, None), - quantile_agg(vec![], Some(2), 0.9, None) - )); - let latency_above_1 = Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(3)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: ExprSemantics::Sql, - }); - assert!(!one_group( - quantile_agg(vec![1], Some(2), 0.5, None), - quantile_agg(vec![1], Some(2), 0.5, Some(latency_above_1)) - )); -} - -/// Equal expressions without a unique key are one group when they return -/// the same rows each time: a keyless scan, a global aggregate, a -/// `without` aggregate. -#[test] -fn equal_keyless_expressions_that_return_the_same_rows_are_one_group() { - let table_scan = |memo: &mut Memo| { - memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - })) - .unwrap() - }; - assert!(one_group(table_scan, table_scan)); - assert!(one_group( - quantile_agg(vec![], Some(2), 0.9, None), - quantile_agg(vec![], Some(2), 0.9, None) - )); - let without = || { - over_scan(|child| { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![], - having: None, - child, - }) - }) - }; - assert!(one_group(without(), without())); -} - -/// `DISTINCT` on every column returns the same rows each time and is one -/// group; a `Dedup` on some columns keeps an unspecified row per key. -#[test] -fn dedup_is_one_group_only_on_every_column() { - let dedup = |cols: Vec| { - over_scan(move |child| Operator::NonASAP(NonASAPOp::Dedup { cols, child })) - }; - assert!(one_group(dedup(vec![]), dedup(vec![]))); - assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); -} - -/// Two unordered `LIMIT 10`s may return different rows: two groups, and -/// so are equal parents over them. -#[test] -fn unordered_limits_and_their_parents_are_different_groups() { - assert!(!one_group( - over_scan(unordered_limit), - over_scan(unordered_limit) - )); - let distinct_over_limit = |memo: &mut Memo| { - let limit = over_scan(unordered_limit)(memo); - memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Dedup { - cols: vec![], - child: limit, - })) - .unwrap() - }; - assert!(!one_group(distinct_over_limit, distinct_over_limit)); -} - -/// A `LIMIT` over a `Sort` with the same partitions keeps the first rows -/// of a fixed order (one group); over a `Sort` partitioned differently, -/// which rows come first depends on input order (two groups). -#[test] -fn limits_over_a_sort_with_their_partitions_are_one_group() { - let limit = |sort_partitions: GroupKeys| { - move |memo: &mut Memo| { - let sorted = over_scan(|child| { - Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(2), - ascending: false, - nulls_first: false, - }], - partition_by: sort_partitions, - child, - }) - })(memo); - memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: sorted, - })) - .unwrap() - } - }; - assert!(one_group( - limit(GroupKeys::default()), - limit(GroupKeys::default()) - )); - assert!(!one_group( - limit(GroupKeys::by(vec![1])), - limit(GroupKeys::by(vec![1])) - )); -} - -/// A projection emitting the literal `value`. -fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { - over_scan(move |child| { - Operator::NonASAP(NonASAPOp::Project { - cols: vec![ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::Literal(ScalarValue::Float64(value)), - }], - child, - }) - }) -} - -/// Signed zeros and different non-finite literals are different groups; -/// JSON's `null` for non-finite floats never decides equality. -#[test] -fn signed_zero_and_non_finite_literals_are_different_groups() { - for (a, b) in [ - (0.0, -0.0), - (-0.0, 0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); - } - assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); -} - /// A KLL state over the latency scan and its p99 readout, both pending: the /// readout's schema is the one its group gets once both are added, and /// nothing is added while deriving it. @@ -625,3 +226,240 @@ fn output_schema_of_a_pending_expression_rejects_what_find_or_add_group_rejects( assert!(memo.output_schema_of_pending_expression(&[], &bad).is_err()); assert!(memo.find_or_add_group(bad).is_err()); } + +/// One readout's required accuracy. +fn required(max_error: f64) -> Vec { + vec![LogicalRequirements { + required_accuracy: Some(crate::stages::RequiredAccuracy { + metric: crate::accuracy::AccuracyMetric::Rank, + max_error, + failure_probability: 1.0, + }), + }] +} + +/// `Memo::set_candidate_input_requirements` +/// replaces $r^{\log}$ of the one candidate it names; two readouts of one +/// state with different required accuracy are two candidates. +#[test] +fn set_candidate_input_requirements_replaces_only_the_named_candidate() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let p90 = || readout(SketchStatistic::Quantile { q: 0.9 }, g2); + let g3 = memo.find_or_add_group(p90()).unwrap(); + let first = try_candidate(&mut memo, g3, p90(), None).unwrap(); + memo.set_candidate_input_requirements(first, required(0.01)); + let second = try_candidate(&mut memo, g3, p90(), None).unwrap(); + assert_ne!(first, second, "an unset copy is a new candidate"); + memo.set_candidate_input_requirements(second, required(0.02)); + let requirements: Vec<_> = memo.groups()[g3.0 as usize] + .candidates + .iter() + .map(|c| c.input_requirements.clone()) + .collect(); + assert_eq!(requirements, vec![required(0.01), required(0.02)]); +} + +/// A merge of KLL states carries the θ of the +/// states it reads. Without one it is rejected; `merge[k=547]` reads only +/// the k = 547 build of its input group. +#[test] +fn a_merge_carries_the_parameters_of_the_states_it_reads() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let kll_state = || state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1); + let g2 = memo.find_or_add_group(kll_state()).unwrap(); + for k in [269, 547] { + try_candidate(&mut memo, g2, kll_state(), kll_k(k)).unwrap(); + } + let merge = Operator::ASAP(crate::ir::ASAPOp::SummaryMerge { + children: vec![g2], + group_by: None, + }); + let g3 = memo.find_or_add_group(merge.clone()).unwrap(); + assert!(matches!( + try_candidate(&mut memo, g3, merge.clone(), None), + Err(CandidateError::LogicalProperties( + LogicalPropertiesError::SketchBuildWithoutParameters + )) + )); + let merged = try_candidate(&mut memo, g3, merge, kll_k(547)).unwrap(); + assert_eq!( + memo.producers(merged, 0), + vec![CandidateForMemoGroupId { + group: g2, + index_in_group: 1 + }] + ); +} + +/// One requirement per input group, or a panic. +#[test] +#[should_panic(expected = "one input requirement per input group")] +fn set_candidate_input_requirements_panics_on_a_length_mismatch() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let kll_state = || state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1); + let g2 = memo.find_or_add_group(kll_state()).unwrap(); + let build = try_candidate(&mut memo, g2, kll_state(), kll_k(269)).unwrap(); + memo.set_candidate_input_requirements(build, Vec::new()); +} + +/// `add_candidate` checks θ against the group's output, for every operator +/// kind: a sketch state's candidates (a build, a chain, a merge) need θ of +/// its algorithm from its list; any other candidate (an exact build, a +/// readout) has none. No Hydra θ is listed yet, so no Hydra build is valid. +#[test] +fn add_candidate_checks_parameters_against_the_groups_output() { + use crate::ir::operator::arguments::Reduction; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::ASAPOp; + use crate::summary_type::{ExactKind, HydraParams, SummaryUpdate}; + use LogicalPropertiesError as Invalid; + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let kll_state = memo.find_or_add_group(state(kll.clone(), g1)).unwrap(); + let kll_chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: g1, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![kll_state], + group_by: None, + }); + // k = 270 is KLL's structure but not in its list. + let parameters = [ + None, + kll_k(269), + Some(SketchParams::Cms { + width: 272, + depth: 5, + }), + Some(SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 1_024, + })), + kll_k(270), + ]; + let no_parameters = Err(Invalid::ParametersOnExactOperator); + let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); + let not_in_list = Err(Invalid::ParametersNotInList); + let exact = [ + Ok(()), + no_parameters, + no_parameters, + no_parameters, + no_parameters, + ]; + // Another algorithm's parameters are not in KLL's list either. + let kll_state_output = [ + needs_parameters, + Ok(()), + not_in_list, + not_in_list, + not_in_list, + ]; + // Per operator: the result for each of `parameters`. + let table = [ + (state(kll, g1), kll_state_output), + ( + state(FieldDataType::Sketch(SketchAlgorithm::HydraCms), g1), + [ + needs_parameters, + not_in_list, + not_in_list, + not_in_list, + not_in_list, + ], + ), + (state(FieldDataType::exact(ExactKind::Sum), g1), exact), + (kll_chain, kll_state_output), + ( + readout(SketchStatistic::Quantile { q: 0.5 }, kll_state), + exact, + ), + (merge, kll_state_output), + ]; + for (operator, by_parameters) in table { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + for (parameters, expected) in parameters.iter().zip(by_parameters) { + let added = try_candidate(&mut memo, group, operator.clone(), parameters.clone()); + let added = match added { + Ok(_) => Ok(()), + Err(CandidateError::LogicalProperties(error)) => Err(error), + Err(error) => panic!("{error}"), + }; + assert_eq!(added, expected, "{:?} {parameters:?}", operator.kind_name()); + } + } +} + +/// A readout returns at most the series scanned below it, the declared +/// distinct values of its state's keys, and one value without keys: over +/// `latency` with 1000 series and 20 declared jobs, a per-series KLL +/// readout returns at most 1000 values, one by `job` 20, one with no key 1. +#[test] +fn max_values_per_evaluation_bounds_a_readout_by_its_states_keys() { + let data = crate::workload::DataWorkload { + sources: [( + "latency".to_string(), + crate::workload::SourceFacts { + series_count: Some(1000), + distinct_values_per_label: [("job".to_string(), 20)].into(), + ..Default::default() + }, + )] + .into(), + ..Default::default() + }; + let mut memo = Memo::new(&data); + let input = memo + .find_or_add_group(scan( + "latency", + &[("job", crate::ir::schema::DataType::Utf8)], + vec![], + )) + .unwrap(); + let job = memo.groups()[input.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == "job") + .expect("the scan lists job"); + for (reduction, values) in [ + (Reduction::PerEntity, 1000), + (Reduction::by(vec![job]), 20), + (Reduction::by(vec![]), 1), + ] { + let state = memo + .find_or_add_group(state_grouped_by( + FieldDataType::Sketch(SketchAlgorithm::Kll), + reduction, + input, + )) + .unwrap(); + let quantile = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.9 }, state)) + .unwrap(); + assert_eq!(memo.max_values_per_evaluation(quantile), Some(values)); + } +} diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 56f1d9232..16125313f 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -2,7 +2,9 @@ //! rule, the logical edges and the candidates dead-candidate elimination //! removed. -use crate::stages::{EvaluationGrid, LogicalEdge, Memo, PrunedLogicalCandidate}; +use std::collections::BTreeSet; + +use crate::stages::{EvaluationGrid, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate}; /// Stage 1, Pass 2's output. #[derive(Debug, Clone)] @@ -18,8 +20,24 @@ pub struct LogicalASAPMemoCandidates { /// Per group, by `MemoGroupId`: the grid a result of the group is /// evaluated on, the coarsest grid holding every evaluation time of the /// queries reaching it (X = `sum_over_time(m[5m])` read every 60 s and - /// every 120 s: 60 s); `None` when those times are not on one known - /// grid. A result of the group materialized across evaluations is + /// every 120 s: 60 s; below a PromQL subquery `[10m:30s]`, its 30 s + /// steps); `None` when those times are not on one known grid (below a + /// subquery without a resolution). A result of the group materialized across evaluations is /// stored at these times. pub evaluation_grid_of_group: Vec>, + /// Indexed by query: the groups this query reaches through a consumer with no + /// error contract for its metric; in these groups the query may only use exact candidates. + /// Q0 = `quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, + /// m[5m])` and Q1 = the same ratio `> 1`, both relative 0.05: Q0 may + /// read the DDSketch readouts, while Q1's comparison has no contract, + /// so its set holds the ratio group and every group below it. (Both + /// list the rows below a sketch build, which has no contract.) A query + /// whose whole-result failure probability has no per-value one ($n_q$ + /// unknown) also lists each readout group another query gave a level, + /// with the groups below it. A group is listed for a query, not + /// pruned: other queries may still read its approximate candidates, so + /// Stage 3 must enforce it per query, in each query's selection + /// context; Pass 2 only lists it. Empty for a query without an accuracy + /// target. + pub exact_only_groups_of_query: Vec>, } diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index b75e6594d..b546a477c 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -1,28 +1,31 @@ -//! The logical side: Stage 0's and Stage 1's memo, by scope (memo → group -//! → candidate → candidate input → edge → Stage 1's output). +//! The logical side: Stage 0's and Stage 1's memo, by scope. +//! +//! - `memo` — the memo and its queries. +//! - `group` — a memo group and the properties its candidates share +//! (column lineage and sign, aggregate coverage). +//! - `candidate` — a group's candidates. +//! - `requirements` — what a candidate requires of each input. +//! - `edge` — the logical edges between candidates. +//! - `memo_candidates` — Stage 1's output. -mod aggregate_coverage; mod candidate; -mod column_properties; mod edge; mod group; mod memo; mod memo_candidates; mod requirements; -mod view_coverage; -pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; pub use candidate::{ - CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, - LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, + listed_build_parameters, CandidateError, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, + OriginOfCandidateForMemoGroup, }; -pub use column_properties::{ColumnLineage, ValueSign}; pub use edge::LogicalEdge; pub use group::{ - ranked_column_of_group, raw_data_samples_of_metric_of_group, GroupProperties, MemoGroup, - MemoGroupError, MemoGroupId, + ranked_column_of_group, raw_data_samples_of_metric_of_group, AggregateCoverage, + AggregateDefinition, ColumnLineage, CoverageError, GroupProperties, MemoGroup, MemoGroupError, + MemoGroupId, ValueSign, }; pub use memo::{Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; pub use requirements::{LogicalRequirements, RequiredAccuracy}; -pub use view_coverage::view_coverage; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index aafdbf2ab..ba1accd63 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -1,20 +1,22 @@ -//! The Stage 0 → Stage 1 boundary: the workload's memo groups, without -//! candidates ([`Memo`]). Inside Stage 1, Pass 1 → Pass 2: the memo with -//! its Pass 1 candidates. The Stage 1 → Stage 2 boundary: the memo of logical groups and their -//! implementation candidates that Stage 1 (Pass 1 and Pass 2) produces and -//! Stage 2 implements (#1047, #1054). The Stage 2 → Stage 3 boundary: the -//! physical candidates Stage 2 builds and Stage 3 selects from. Stage -//! crates never depend on each other, so the types more than one stage reads -//! live here. +//! The types one planning stage hands the next, so that stage crates never +//! depend on each other: +//! +//! - `logical` — Stage 0's and Stage 1's memo ([`Memo`]: groups, queries, +//! candidates and their edges), and Stage 1's output +//! ([`LogicalASAPMemoCandidates`]) that Stage 2 implements. +//! - `physical` — Stage 2's output ([`PhysicalASAPMemoCandidates`]): the +//! physical candidates and edges Stage 3 selects from. +//! - `pruning` — what each stage removed and why ([`PrunedCandidate`], +//! [`PruneReason`]), one shape for every stage. mod logical; mod physical; mod pruning; pub use logical::{ - ranked_column_of_group, raw_data_samples_of_metric_of_group, view_coverage, AggregateCoverage, - AggregateDefinition, CandidateError, CandidateExpression, CandidateForMemoGroup, - CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, + listed_build_parameters, ranked_column_of_group, raw_data_samples_of_metric_of_group, + AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, + CandidateForMemoGroup, CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index c5891c112..92bd5f997 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -1,5 +1,5 @@ //! Candidate scope: each logical candidate's physical alternatives, one per -//! valid $p^{\mathrm{phys}} = (\tau, \ell)$ (#1200). +//! valid $p^{\mathrm{phys}} = (\tau, \ell)$. use crate::ir::properties::ExecutionTiming; use crate::ir::{ASAPOp, Operator}; diff --git a/crates/types/src/stages/physical/edge.rs b/crates/types/src/stages/physical/edge.rs index 7df48ad9a..07f30e68b 100644 --- a/crates/types/src/stages/physical/edge.rs +++ b/crates/types/src/stages/physical/edge.rs @@ -1,5 +1,5 @@ //! Edge scope: the physical edges $\mathcal E$ with $\alpha^{\log}$ and -//! $\alpha^{\mathrm{phys}}$ (#1201). +//! $\alpha^{\mathrm{phys}}$. use crate::stages::PhysicalCandidateId; diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs deleted file mode 100644 index 502592071..000000000 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ /dev/null @@ -1,421 +0,0 @@ -//! Stage 2's output: each logical candidate's physical alternatives -//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and -//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and -//! $\alpha^{\mathrm{phys}}$ (#1200, #1201). Stage 3 selects among them; -//! Stage 2 prices nothing. - -use std::collections::HashMap; - -use crate::ir::QueryRoot; -use crate::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, PrunedCandidate, -}; - -/// Stage 2's output, which Stage 3 selects from. -#[derive(Debug, Clone)] -pub struct PhysicalASAPMemoCandidates { - /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, - /// $r^{\log}_{c,i}$, and the logical edges. - pub logical_candidates: LogicalASAPMemoCandidates, - /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. - pub physical_candidates: Vec, - /// The alternatives Stage 2 pruned, in pruning order, each as the - /// [`PhysicalCandidate`] it would have been: candidates are numbered - /// after pruning, so a pruned one has no [`PhysicalCandidateId`]. - pub pruned_candidates: Vec>, - /// $\mathcal E$, grouped by (consumer, input); indexed by - /// [`PhysicalEdgeId`]. - edges: Vec, - /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). - admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, - /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. - admissible_edges_out_of_candidate: Vec>, -} - -impl PhysicalASAPMemoCandidates { - /// Stage 2's output over `physical_candidates` and `edges` (grouped by - /// (consumer, input)), with the admissible-edge indexes built once. - /// Query roots are read from `logical_candidates`, not copied. - pub fn new( - logical_candidates: LogicalASAPMemoCandidates, - physical_candidates: Vec, - edges: Vec, - pruned_candidates: Vec>, - ) -> Self { - let _ = ( - logical_candidates, - physical_candidates, - edges, - pruned_candidates, - ); - todo!("#1207 PR 5: index the admissible edges") - } - - /// $\mathcal E$: every edge, in (consumer, input) order. - pub fn edges(&self) -> impl Iterator + '_ { - self.edges - .iter() - .enumerate() - .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) - } - - /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input - /// `input_index_of_consumer` of `consumer` - /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). - pub fn admissible_edges_into( - &self, - consumer: PhysicalCandidateId, - input_index_of_consumer: usize, - ) -> &[PhysicalEdgeId] { - let _ = ( - consumer, - input_index_of_consumer, - &self.admissible_edges_into_input, - ); - todo!("#1207 PR 5") - } - - /// The admissible edges out of `producer`: every consumer that may share it. - pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { - let _ = (producer, &self.admissible_edges_out_of_candidate); - todo!("#1207 PR 5") - } -} - -/// The output references of `query_roots`: each operator reference, in -/// order (a scalar root's in `map_operator_refs` order), with the index of -/// the query whose root it is. -pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, C)> { - let mut references = Vec::new(); - for (query, root) in query_roots.iter().enumerate() { - match root { - QueryRoot::Operator(reference) => references.push((query, reference.clone())), - QueryRoot::Scalar(expression) => { - expression.map_operator_refs(&mut |reference| { - references.push((query, reference.clone())) - }); - } - } - } - references -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::properties::ExecutionTiming; - use crate::ir::{ASAPOp, Operator}; - use crate::stages::{ - producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, - Lifetime, PhysicalProperties, - }; - use crate::workload::DataWorkload; - - use crate::plan::materialized_view::RefreshWindow; - - use crate::ir::operator::arguments::{Reduction, Source}; - - use crate::ir::scalar::expr_ir::ColumnRef; - - use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; - use crate::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; - - use crate::ir::{NonASAPOp, TimeRangeKind}; - - use crate::stages::StorageMedium; - - use crate::stages::physical::execution_context::EvaluationGrid; - - use crate::stages::{ - CandidateExpression, CandidateForMemoGroup, LogicalASAPMemoCandidates, LogicalProperties, - LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, - }; - - use ExecutionTiming::{IngestionTime, QueryTime}; - - /// The KLL build over the scan: not a chain. - const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(1), - index_in_group: 0, - }; - - /// The chain of tumbling windows 5 s wide over the scan. - const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(2), - index_in_group: 0, - }; - - /// The chain of tumbling windows 10 s wide over the scan. - const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(3), - index_in_group: 0, - }; - - /// X = the scan's last 5 minutes (`m[5m]`): a result that depends on - /// the evaluation time and is not a chain. - const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(4), - index_in_group: 0, - }; - - /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains - /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last - /// 5 minutes, each with its own expression as its one candidate. Built - /// through `find_or_add_group` and `add_candidate`. - fn memo() -> Memo { - let mut memo = Memo::new(&DataWorkload::default()); - let scan = memo - .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); - let build = Operator::ASAP(ASAPOp::SummaryAgg { - child: scan, - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }); - let chain = |tumbling_window_width_ms| { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: scan, - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - tumbling_window_width_ms, - }) - }; - for operator in [build, chain(5_000), chain(10_000)] { - let group = memo.find_or_add_group(operator.clone()).unwrap(); - let expression = CandidateExpression { operator }; - let candidate = CandidateForMemoGroup { - logical_properties: LogicalProperties::new( - &expression, - Some(SketchParams::Kll { k: 269 }), - ) - .unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; - memo.add_candidate(group, candidate).unwrap(); - } - let operator = Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - }); - let group = memo.find_or_add_group(operator.clone()).unwrap(); - let expression = CandidateExpression { operator }; - let candidate = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; - memo.add_candidate(group, candidate).unwrap(); - memo - } - - /// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; - /// `delivers_on` reads nothing else. - fn candidates( - memo: Memo, - evaluation_grid_of_group: Vec>, - ) -> PhysicalASAPMemoCandidates { - PhysicalASAPMemoCandidates { - logical_candidates: LogicalASAPMemoCandidates { - memo, - edges: vec![], - pruned: vec![], - evaluation_grid_of_group, - }, - physical_candidates: vec![], - pruned_candidates: vec![], - edges: vec![], - admissible_edges_into_input: HashMap::new(), - admissible_edges_out_of_candidate: vec![], - } - } - - fn candidate( - logical_candidate_id: CandidateForMemoGroupId, - timing: ExecutionTiming, - lifetime: Lifetime, - ) -> PhysicalCandidate { - PhysicalCandidate { - logical_candidate_id, - physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), - } - } - - fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { - ExecutionContext::Evaluation { - grid: EvaluationGrid { - period_ms, - phase_ms, - }, - } - } - - fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { - ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) - } - - const MEMORY: StorageMedium = StorageMedium::Memory; - - /// A `None` producer runs in the consumer's context and delivers on - /// every one; a `OneEvaluation` producer only on an evaluation context, - /// the one evaluation its result belongs to. Neither reads the memo. - #[test] - fn delivers_on_by_lifetime_without_windows() { - let candidates = candidates(Memo::new(&DataWorkload::default()), vec![]); - let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); - let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); - let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); - for (producer, context, delivers) in [ - (&unmaterialized, evaluation(7_000, 3_000), true), - (&unmaterialized, ingestion(5_000, 0), true), - (&unmaterialized, materialized, true), - (&one_evaluation, evaluation(7_000, 3_000), true), - (&one_evaluation, ingestion(5_000, 0), false), - (&one_evaluation, materialized, false), - ] { - assert_eq!( - producer.delivers_on(&candidates, &context), - delivers, - "{:?} in {context:?}", - producer.physical_properties - ); - } - } - - /// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase - /// 0) delivers when $w$ divides the context's period (or refresh-window - /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s - /// serves both. - #[test] - fn delivers_on_a_chain_when_its_width_divides_the_context_times() { - let candidates = candidates(memo(), vec![None; 5]); - let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); - let ingestion_10s = candidate( - WINDOWS_10S, - IngestionTime, - Lifetime::AcrossEvaluations(MEMORY), - ); - for (producer, context, delivers) in [ - // Evaluation grids: w divides period and phase. - (&across_5s, evaluation(10_000, 0), true), - (&across_5s, evaluation(10_000, 5_000), true), - (&across_5s, evaluation(10_000, 2_000), false), - (&across_5s, evaluation(7_000, 0), false), - (&ingestion_10s, evaluation(10_000, 0), true), - (&ingestion_10s, evaluation(10_000, 5_000), false), - // Refresh windows: w divides width and phase. - (&across_5s, ingestion(5_000, 0), true), - (&across_5s, ingestion(10_000, 5_000), true), - (&across_5s, ingestion(10_000, 2_500), false), - (&ingestion_10s, ingestion(5_000, 0), false), - ] { - assert_eq!( - producer.delivers_on(&candidates, &context), - delivers, - "{:?} {:?} in {context:?}", - producer.logical_candidate_id, - producer.physical_properties - ); - } - } - - /// A stored result that is not a chain delivers when its group's - /// evaluation grid holds every time of the context: X = - /// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is - /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or - /// another phase. Without a known grid it serves none. - #[test] - fn delivers_on_a_stored_result_on_its_group_grid() { - let grid_60s = Some(EvaluationGrid { - period_ms: 60_000, - phase_ms: 0, - }); - let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); - let without_grid = candidates(memo(), vec![None; 5]); - let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); - for (candidates, context, delivers) in [ - (&on_grid, evaluation(60_000, 0), true), - (&on_grid, evaluation(120_000, 0), true), - (&on_grid, evaluation(120_000, 60_000), true), - (&on_grid, evaluation(30_000, 0), false), - (&on_grid, evaluation(60_000, 15_000), false), - (&on_grid, ingestion(60_000, 0), true), - (&without_grid, evaluation(60_000, 0), false), - ] { - assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); - } - } - - /// `producer_satisfies_physical_requirements` over every valid producer state and every valid - /// consumer state: a query-time producer serves only query-time - /// consumers, an ingestion-time producer that is not materialized only - /// ingestion-time consumers, and one materialized across evaluations - /// serves both. Chain widths do not matter: alignment is Stage 3's - /// per-context check. - #[test] - fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { - let states = |medium| { - [ - candidate(BUILD, QueryTime, Lifetime::None), - candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), - candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), - candidate(BUILD, IngestionTime, Lifetime::None), - candidate( - WINDOWS_5S, - IngestionTime, - Lifetime::AcrossEvaluations(medium), - ), - ] - }; - // Rows: producer state; columns: consumer state, in `states` order. - let expected = [ - [true, true, true, false, false], - [true, true, true, false, false], - [true, true, true, false, false], - [false, false, false, true, true], - [true, true, true, true, true], - ]; - for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { - for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { - for (producer, row) in states(producer_medium).iter().zip(expected) { - for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { - assert_eq!( - producer_satisfies_physical_requirements( - producer, - consumer.physical_properties.timing() - ), - satisfies, - "{:?} -> {:?}", - producer.physical_properties, - consumer.physical_properties - ); - } - } - } - } - } -} diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs new file mode 100644 index 000000000..523021a54 --- /dev/null +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -0,0 +1,105 @@ +//! Stage 2's output: each logical candidate's physical alternatives +//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and +//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$. Stage 3 selects among them; +//! Stage 2 prices nothing. + +use std::collections::HashMap; + +use crate::ir::QueryRoot; +use crate::stages::{ + LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + PhysicalEdgeId, PrunedCandidate, +}; + +/// Stage 2's output, which Stage 3 selects from. +#[derive(Debug, Clone)] +pub struct PhysicalASAPMemoCandidates { + /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, + /// $r^{\log}_{c,i}$, and the logical edges. + pub logical_candidates: LogicalASAPMemoCandidates, + /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. + pub physical_candidates: Vec, + /// The alternatives Stage 2 pruned, in pruning order, each as the + /// [`PhysicalCandidate`] it would have been: candidates are numbered + /// after pruning, so a pruned one has no [`PhysicalCandidateId`]. + pub pruned_candidates: Vec>, + /// $\mathcal E$, grouped by (consumer, input); indexed by + /// [`PhysicalEdgeId`]. + edges: Vec, + /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). + admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, + /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. + admissible_edges_out_of_candidate: Vec>, +} + +impl PhysicalASAPMemoCandidates { + /// Stage 2's output over `physical_candidates` and `edges` (grouped by + /// (consumer, input)), with the admissible-edge indexes built once. + /// Query roots are read from `logical_candidates`, not copied. + pub fn new( + logical_candidates: LogicalASAPMemoCandidates, + physical_candidates: Vec, + edges: Vec, + pruned_candidates: Vec>, + ) -> Self { + let _ = ( + logical_candidates, + physical_candidates, + edges, + pruned_candidates, + ); + todo!("#1207 PR 5: index the admissible edges") + } + + /// $\mathcal E$: every edge, in (consumer, input) order. + pub fn edges(&self) -> impl Iterator + '_ { + self.edges + .iter() + .enumerate() + .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) + } + + /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input + /// `input_index_of_consumer` of `consumer` + /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). + pub fn admissible_edges_into( + &self, + consumer: PhysicalCandidateId, + input_index_of_consumer: usize, + ) -> &[PhysicalEdgeId] { + let _ = ( + consumer, + input_index_of_consumer, + &self.admissible_edges_into_input, + ); + todo!("#1207 PR 5") + } + + /// The admissible edges out of `producer`: every consumer that may share it. + pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { + let _ = (producer, &self.admissible_edges_out_of_candidate); + todo!("#1207 PR 5") + } +} + +/// The output references of `query_roots`: each operator reference, in +/// order (a scalar root's in `map_operator_refs` order), with the index of +/// the query whose root it is. +pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, C)> { + let mut references = Vec::new(); + for (query, root) in query_roots.iter().enumerate() { + match root { + QueryRoot::Operator(reference) => references.push((query, reference.clone())), + QueryRoot::Scalar(expression) => { + expression.map_operator_refs(&mut |reference| { + references.push((query, reference.clone())) + }); + } + } + } + references +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs new file mode 100644 index 000000000..723104d35 --- /dev/null +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -0,0 +1,304 @@ +use super::*; +use crate::ir::properties::ExecutionTiming; +use crate::ir::{ASAPOp, Operator}; +use crate::stages::{ + producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, Lifetime, + PhysicalProperties, +}; +use crate::workload::DataWorkload; + +use crate::plan::materialized_view::RefreshWindow; + +use crate::ir::operator::arguments::{Reduction, Source}; + +use crate::ir::scalar::expr_ir::ColumnRef; + +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; + +use crate::ir::{NonASAPOp, TimeRangeKind}; + +use crate::stages::StorageMedium; + +use crate::stages::physical::execution_context::EvaluationGrid; + +use crate::stages::{ + CandidateForMemoGroup, LogicalASAPMemoCandidates, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, +}; + +use ExecutionTiming::{IngestionTime, QueryTime}; + +/// The KLL build over the scan: not a chain. +const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(1), + index_in_group: 0, +}; + +/// The chain of tumbling windows 5 s wide over the scan. +const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(2), + index_in_group: 0, +}; + +/// The chain of tumbling windows 10 s wide over the scan. +const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(3), + index_in_group: 0, +}; + +/// X = the scan's last 5 minutes (`m[5m]`): a result that depends on +/// the evaluation time and is not a chain. +const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(4), + index_in_group: 0, +}; + +/// Group 1, KLL states over the scan group 0, groups 2 and 3, chains +/// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last +/// 5 minutes, each with its own expression as its one candidate. Built +/// through `find_or_add_group` and `add_candidate`. +fn memo() -> Memo { + let mut memo = Memo::new(&DataWorkload::default()); + let scan = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let chain = |tumbling_window_width_ms| { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms, + }) + }; + for operator in [build, chain(5_000), chain(10_000)] { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let candidate = CandidateForMemoGroup::new( + operator, + Some(SketchParams::Kll { k: 269 }), + OriginOfCandidateForMemoGroup::Pass1, + vec![], + ); + memo.add_candidate(group, candidate).unwrap(); + } + let operator = Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let candidate = + CandidateForMemoGroup::new(operator, None, OriginOfCandidateForMemoGroup::Pass1, vec![]); + memo.add_candidate(group, candidate).unwrap(); + memo +} + +/// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; +/// `delivers_on` reads nothing else. +fn candidates( + memo: Memo, + evaluation_grid_of_group: Vec>, +) -> PhysicalASAPMemoCandidates { + PhysicalASAPMemoCandidates { + logical_candidates: LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group, + exact_only_groups_of_query: vec![], + }, + physical_candidates: vec![], + pruned_candidates: vec![], + edges: vec![], + admissible_edges_into_input: HashMap::new(), + admissible_edges_out_of_candidate: vec![], + } +} + +fn candidate( + logical_candidate_id: CandidateForMemoGroupId, + timing: ExecutionTiming, + lifetime: Lifetime, +) -> PhysicalCandidate { + PhysicalCandidate { + logical_candidate_id, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + } +} + +fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms, + }, + } +} + +fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) +} + +const MEMORY: StorageMedium = StorageMedium::Memory; + +/// A `None` producer runs in the consumer's context and delivers on +/// every one; a `OneEvaluation` producer only on an evaluation context, +/// the one evaluation its result belongs to. Neither reads the memo. +#[test] +fn delivers_on_by_lifetime_without_windows() { + let candidates = candidates(Memo::new(&DataWorkload::default()), vec![]); + let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); + let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); + let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); + for (producer, context, delivers) in [ + (&unmaterialized, evaluation(7_000, 3_000), true), + (&unmaterialized, ingestion(5_000, 0), true), + (&unmaterialized, materialized, true), + (&one_evaluation, evaluation(7_000, 3_000), true), + (&one_evaluation, ingestion(5_000, 0), false), + (&one_evaluation, materialized, false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} in {context:?}", + producer.physical_properties + ); + } +} + +/// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase +/// 0) delivers when $w$ divides the context's period (or refresh-window +/// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s +/// serves both. +#[test] +fn delivers_on_a_chain_when_its_width_divides_the_context_times() { + let candidates = candidates(memo(), vec![None; 5]); + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let ingestion_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for (producer, context, delivers) in [ + // Evaluation grids: w divides period and phase. + (&across_5s, evaluation(10_000, 0), true), + (&across_5s, evaluation(10_000, 5_000), true), + (&across_5s, evaluation(10_000, 2_000), false), + (&across_5s, evaluation(7_000, 0), false), + (&ingestion_10s, evaluation(10_000, 0), true), + (&ingestion_10s, evaluation(10_000, 5_000), false), + // Refresh windows: w divides width and phase. + (&across_5s, ingestion(5_000, 0), true), + (&across_5s, ingestion(10_000, 5_000), true), + (&across_5s, ingestion(10_000, 2_500), false), + (&ingestion_10s, ingestion(5_000, 0), false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} {:?} in {context:?}", + producer.logical_candidate_id, + producer.physical_properties + ); + } +} + +/// A stored result that is not a chain delivers when its group's +/// evaluation grid holds every time of the context: X = +/// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is +/// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or +/// another phase. Without a known grid it serves none. +#[test] +fn delivers_on_a_stored_result_on_its_group_grid() { + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); + let without_grid = candidates(memo(), vec![None; 5]); + let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + for (candidates, context, delivers) in [ + (&on_grid, evaluation(60_000, 0), true), + (&on_grid, evaluation(120_000, 0), true), + (&on_grid, evaluation(120_000, 60_000), true), + (&on_grid, evaluation(30_000, 0), false), + (&on_grid, evaluation(60_000, 15_000), false), + (&on_grid, ingestion(60_000, 0), true), + (&without_grid, evaluation(60_000, 0), false), + ] { + assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + } +} + +/// `producer_satisfies_physical_requirements` over every valid producer state and every valid +/// consumer state: a query-time producer serves only query-time +/// consumers, an ingestion-time producer that is not materialized only +/// ingestion-time consumers, and one materialized across evaluations +/// serves both. Chain widths do not matter: alignment is Stage 3's +/// per-context check. +#[test] +fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { + let states = |medium| { + [ + candidate(BUILD, QueryTime, Lifetime::None), + candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), + candidate(BUILD, IngestionTime, Lifetime::None), + candidate( + WINDOWS_5S, + IngestionTime, + Lifetime::AcrossEvaluations(medium), + ), + ] + }; + // Rows: producer state; columns: consumer state, in `states` order. + let expected = [ + [true, true, true, false, false], + [true, true, true, false, false], + [true, true, true, false, false], + [false, false, false, true, true], + [true, true, true, true, true], + ]; + for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (producer, row) in states(producer_medium).iter().zip(expected) { + for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { + assert_eq!( + producer_satisfies_physical_requirements( + producer, + consumer.physical_properties.timing() + ), + satisfies, + "{:?} -> {:?}", + producer.physical_properties, + consumer.physical_properties + ); + } + } + } + } +} diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index b733ee832..2b3728d4d 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -1,6 +1,12 @@ //! The Stage 2 → Stage 3 boundary: the physical candidates and edges Stage 2 -//! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201), -//! by scope (execution context → candidate → edge → Stage 2's output). +//! builds from the memo and Stage 3 selects from, by scope (execution +//! context → candidate → edge → Stage 2's output). +//! +//! - `execution_context` — the contexts a candidate runs in. +//! - `candidate` — a logical candidate's physical alternatives. +//! - `properties` — a candidate's physical properties (timing, lifetime). +//! - `edge` — the physical edges between candidates. +//! - `memo_candidates` — Stage 2's output. mod candidate; mod edge; diff --git a/crates/types/src/stages/physical/properties.rs b/crates/types/src/stages/physical/properties/mod.rs similarity index 67% rename from crates/types/src/stages/physical/properties.rs rename to crates/types/src/stages/physical/properties/mod.rs index 3cd8f2a40..e37dad27d 100644 --- a/crates/types/src/stages/physical/properties.rs +++ b/crates/types/src/stages/physical/properties/mod.rs @@ -1,5 +1,5 @@ //! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell)$ -//! (#1200; concepts doc `memo-groups-properties-requirements.md`, sections 1 +//! (concepts doc `memo-groups-properties-requirements.md`, sections 1 //! and 3): when it runs ([`ExecutionTiming`]) and how long its output lives //! ([`Lifetime`]). There is no schedule property: a `None` or `OneEvaluation` //! output runs on the schedule of the context κ that runs it, and a tumbling @@ -89,39 +89,4 @@ impl PhysicalProperties { } #[cfg(test)] -mod tests { - use super::*; - use ExecutionTiming::{IngestionTime, QueryTime}; - - /// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for - /// both media, and keeps what it was given. - #[test] - fn new_over_every_timing_and_lifetime() { - for medium in [StorageMedium::Memory, StorageMedium::Disk] { - for (timing, lifetime, valid) in [ - (QueryTime, Lifetime::None, true), - (QueryTime, Lifetime::OneEvaluation(medium), true), - (QueryTime, Lifetime::AcrossEvaluations(medium), true), - (IngestionTime, Lifetime::None, true), - (IngestionTime, Lifetime::OneEvaluation(medium), false), - (IngestionTime, Lifetime::AcrossEvaluations(medium), true), - ] { - let properties = PhysicalProperties::new(timing, lifetime); - match valid { - true => { - let properties = properties.unwrap(); - assert_eq!( - (properties.timing(), properties.lifetime()), - (timing, lifetime) - ); - assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); - } - false => assert_eq!( - properties, - Err(PhysicalPropertiesError::OneEvaluationAtIngestion) - ), - } - } - } - } -} +mod tests; diff --git a/crates/types/src/stages/physical/properties/tests.rs b/crates/types/src/stages/physical/properties/tests.rs new file mode 100644 index 000000000..5a0ad7f10 --- /dev/null +++ b/crates/types/src/stages/physical/properties/tests.rs @@ -0,0 +1,34 @@ +use super::*; +use ExecutionTiming::{IngestionTime, QueryTime}; + +/// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for +/// both media, and keeps what it was given. +#[test] +fn new_over_every_timing_and_lifetime() { + for medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (timing, lifetime, valid) in [ + (QueryTime, Lifetime::None, true), + (QueryTime, Lifetime::OneEvaluation(medium), true), + (QueryTime, Lifetime::AcrossEvaluations(medium), true), + (IngestionTime, Lifetime::None, true), + (IngestionTime, Lifetime::OneEvaluation(medium), false), + (IngestionTime, Lifetime::AcrossEvaluations(medium), true), + ] { + let properties = PhysicalProperties::new(timing, lifetime); + match valid { + true => { + let properties = properties.unwrap(); + assert_eq!( + (properties.timing(), properties.lifetime()), + (timing, lifetime) + ); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + false => assert_eq!( + properties, + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ), + } + } + } +} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index c389921c2..6a8fdeef4 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,13 +1,14 @@ -//! What a stage removed, and why: one shape for every stage (#1047 P6). -//! Each stage removes only what its own knowledge rules out (option C): -//! Stage 2 what cannot be built, run or read by a query output. Stage 0 fails with an -//! error, Pass 1 records nothing it leaves out (#1205 will show it), and -//! Pass 2 records its eliminated candidates with their own causes -//! (`PrunedLogicalCandidate`). Stage 3 prunes before -//! its selection MILP (dominated and uncostable candidates); its -//! rejections of whole plans keep their own cause (`RejectionCause` in plan -//! selection). The planner does not check whether an executor implements an -//! operator or summary type, so no reason names it. +//! What a stage removed, and why: one shape for every stage. Each stage +//! removes only what its own knowledge rules out. Pass 2 records its +//! eliminated logical candidates with their causes ([`PrunedLogicalCandidate`], +//! [`PruneCause`], [`RootPruneReason`]); Stage 2 records the physical +//! candidates that cannot be built, run or read by a query output +//! ([`PrunedCandidate`], [`PruneReason`]); Stage 3 prunes dominated and +//! uncostable candidates before its selection, and its rejections of whole +//! plans keep their own cause (`RejectionCause` in plan selection). Stage 0 +//! fails with an error and Pass 1 records nothing it leaves out. The planner +//! does not check whether an executor implements an operator or summary +//! type, so no reason names it. use crate::accuracy::AccuracyMetric; use crate::stages::{CandidateForMemoGroupId, PhysicalCandidateId, RequiredAccuracy}; @@ -22,12 +23,13 @@ pub struct PrunedLogicalCandidate { pub causes: Vec, } -/// One path from the candidate, through consumers that are themselves +/// One path from the candidate, through candidates that are themselves /// eliminated, to the edge that fails. One shortest path per failing edge. #[derive(Debug, Clone)] pub struct PruneCause { - /// The eliminated consumers on the path, nearest first. - pub through_pruned_consumers: Vec, + /// The eliminated candidates on the path, nearest first: consumers + /// above, or, for a candidate cut off from an input, producers below. + pub through_pruned_candidates: Vec, pub root: RootPruneReason, } @@ -42,12 +44,52 @@ pub enum RootPruneReason { input: usize, required: RequiredAccuracy, }, - /// The consumer's error contract carries none of the producer's metrics. + /// The metric rule: the producer is a readout whose metric the + /// consumer does not accept on that input; or, on a readout's own input, + /// the readout's metric is not one its output accepts. MetricRule { consumer: CandidateForMemoGroupId, input: usize, producer_metric: AccuracyMetric, - consumer_carries: Vec, + /// The metrics accepted on that input (on a readout's own input: at + /// its output). + accepted_metrics: Vec, + }, + /// The consumer is a top-k readout whose `k` exceeds the producer's + /// heap size: the heap cannot hold the k ranked keys, whatever accuracy + /// is required. + HeapSmallerThanTopK { + consumer: CandidateForMemoGroupId, + input: usize, + k: usize, + }, + /// The producer is a readout with an accuracy metric but no guarantee + /// evaluable at the required failure probability (an HLL distinct + /// count, whose failure probability is unknown), or with no guarantee + /// at all (UnivMon's distinct count), read by a query with an accuracy + /// target: its error has no bound. + NoGuarantee { + consumer: CandidateForMemoGroupId, + input: usize, + }, + /// The readout `consumer` is reached by query `query`, whose target sets + /// a failure probability over the whole result while the number of + /// values one evaluation returns ($n_q$) is unknown: no per-value + /// failure probability follows, so no state meets it; and no other + /// query reaching it gives it a level. + ValuesPerEvaluationUnknown { + consumer: CandidateForMemoGroupId, + query: usize, + }, + /// The producer is a candidate over a tumbling-window chain whose width + /// does not divide the consumer group's evaluation period (or the group + /// has no known evaluation grid): the chain's windows do not end at the + /// consumer's evaluation times. + ChainWidthDoesNotDivideEvaluationGrid { + consumer: CandidateForMemoGroupId, + input: usize, + tumbling_window_width_ms: u64, + evaluation_period_ms: Option, }, /// The candidate never had a consumer and is not a query output. A /// candidate whose consumers were eliminated is traced to their causes. diff --git a/crates/types/src/summary_type/capability.rs b/crates/types/src/summary_type/capability.rs index b615192e9..cad6860a7 100644 --- a/crates/types/src/summary_type/capability.rs +++ b/crates/types/src/summary_type/capability.rs @@ -208,7 +208,7 @@ impl SummaryTypeCapability { }, // A sketch algorithm's row is sketchlib's. HydraKll merges as // that row says: `Hydra::merge` requires the same k and m and - // otherwise returns an error (asap_sketchlib #251). + // otherwise returns an error (asap_sketchlib). FieldDataType::Sketch(algorithm) => { let row = algorithm.capability(); return Self { diff --git a/crates/types/src/summary_type/parameter_lists.rs b/crates/types/src/summary_type/parameter_lists/mod.rs similarity index 75% rename from crates/types/src/summary_type/parameter_lists.rs rename to crates/types/src/summary_type/parameter_lists/mod.rs index 5ca4c3b62..d1ecd65e4 100644 --- a/crates/types/src/summary_type/parameter_lists.rs +++ b/crates/types/src/summary_type/parameter_lists/mod.rs @@ -7,10 +7,10 @@ use super::{SketchAlgorithm, SketchParams}; /// error of each, for a statistic at a failure probability, is its guarantee /// (`asap_accuracy::theoretical_guarantee`). /// -/// The widths, `k`s, precisions, `alpha`s and heap sizes are today's sizing -/// (`asap_accuracy`'s test-only `guarantee::sizing_oracle`) over the errors ε = 0.1, -/// 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, without -/// repeats; an `asap_accuracy` test keeps them equal. Distinct counts (KMV, +/// The widths, `k`s, precisions, `alpha`s and heap sizes span the errors +/// 0.1 down to 0.001, smallest structure first, without repeats; an +/// `asap_accuracy` test checks every entry has a guarantee at δ = 0.01. +/// Distinct counts (KMV, /// and UnivMon's distinct-count shapes) stop at ε = 0.01. Where δ is set by /// a structural parameter, the list covers the δ levels with it: Count-Min /// lists each width with each depth of [`CMS_DEPTHS`], Count Sketch with @@ -242,66 +242,4 @@ const fn univmon_parameters() -> [SketchParams; 40] { } #[cfg(test)] -mod tests { - use super::*; - - /// Each predefined list holds one parameter type, without repeats, and - /// is empty exactly for the algorithms without an error model. Count-Min - /// lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, - /// 81, 117 and 155, and a - /// heap variant each of those with every heap size; UnivMon's list has 40 - /// entries. - #[test] - fn parameter_list_holds_its_algorithm_without_repeats() { - let listed = [ - SketchAlgorithm::Kll, - SketchAlgorithm::DDSketch, - SketchAlgorithm::Hll, - SketchAlgorithm::Kmv, - SketchAlgorithm::Cms, - SketchAlgorithm::CountSketch, - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - SketchAlgorithm::UnivMon, - ]; - for algorithm in listed.iter().copied().chain([ - SketchAlgorithm::HydraCms, - SketchAlgorithm::HydraCountSketch, - SketchAlgorithm::HydraKll, - SketchAlgorithm::HllHip, - ]) { - let list = parameter_list(algorithm); - assert_eq!( - list.is_empty(), - !listed.contains(&algorithm), - "{algorithm:?}" - ); - for (i, parameters) in list.iter().enumerate() { - assert_eq!( - std::mem::discriminant(parameters), - std::mem::discriminant(&list[0]) - ); - assert!(!list[..i].contains(parameters), "{parameters:?} repeats"); - } - } - let depths = |algorithm| -> Vec { - parameter_list(algorithm) - .iter() - .take(4) - .map(|parameters| match parameters { - SketchParams::Cms { depth, .. } - | SketchParams::CountSketch { depth, .. } - | SketchParams::CmsWithHeap { depth, .. } - | SketchParams::CountSketchWithHeap { depth, .. } => *depth, - other => panic!("{other:?}"), - }) - .collect() - }; - assert_eq!(depths(SketchAlgorithm::Cms), [5, 7, 10, 12]); - assert_eq!(depths(SketchAlgorithm::CountSketch), [47, 81, 117, 155]); - assert_eq!(depths(SketchAlgorithm::CmsWithHeap), [5, 5, 5, 5]); - assert_eq!(parameter_list(SketchAlgorithm::Cms).len(), 28); - assert_eq!(parameter_list(SketchAlgorithm::CmsWithHeap).len(), 280); - assert_eq!(parameter_list(SketchAlgorithm::UnivMon).len(), 40); - } -} +mod tests; diff --git a/crates/types/src/summary_type/parameter_lists/tests.rs b/crates/types/src/summary_type/parameter_lists/tests.rs new file mode 100644 index 000000000..39c1f4513 --- /dev/null +++ b/crates/types/src/summary_type/parameter_lists/tests.rs @@ -0,0 +1,61 @@ +use super::*; + +/// Each predefined list holds one parameter type, without repeats, and +/// is empty exactly for the algorithms without an error model. Count-Min +/// lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, +/// 81, 117 and 155, and a +/// heap variant each of those with every heap size; UnivMon's list has 40 +/// entries. +#[test] +fn parameter_list_holds_its_algorithm_without_repeats() { + let listed = [ + SketchAlgorithm::Kll, + SketchAlgorithm::DDSketch, + SketchAlgorithm::Hll, + SketchAlgorithm::Kmv, + SketchAlgorithm::Cms, + SketchAlgorithm::CountSketch, + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + SketchAlgorithm::UnivMon, + ]; + for algorithm in listed.iter().copied().chain([ + SketchAlgorithm::HydraCms, + SketchAlgorithm::HydraCountSketch, + SketchAlgorithm::HydraKll, + SketchAlgorithm::HllHip, + ]) { + let list = parameter_list(algorithm); + assert_eq!( + list.is_empty(), + !listed.contains(&algorithm), + "{algorithm:?}" + ); + for (i, parameters) in list.iter().enumerate() { + assert_eq!( + std::mem::discriminant(parameters), + std::mem::discriminant(&list[0]) + ); + assert!(!list[..i].contains(parameters), "{parameters:?} repeats"); + } + } + let depths = |algorithm| -> Vec { + parameter_list(algorithm) + .iter() + .take(4) + .map(|parameters| match parameters { + SketchParams::Cms { depth, .. } + | SketchParams::CountSketch { depth, .. } + | SketchParams::CmsWithHeap { depth, .. } + | SketchParams::CountSketchWithHeap { depth, .. } => *depth, + other => panic!("{other:?}"), + }) + .collect() + }; + assert_eq!(depths(SketchAlgorithm::Cms), [5, 7, 10, 12]); + assert_eq!(depths(SketchAlgorithm::CountSketch), [47, 81, 117, 155]); + assert_eq!(depths(SketchAlgorithm::CmsWithHeap), [5, 5, 5, 5]); + assert_eq!(parameter_list(SketchAlgorithm::Cms).len(), 28); + assert_eq!(parameter_list(SketchAlgorithm::CmsWithHeap).len(), 280); + assert_eq!(parameter_list(SketchAlgorithm::UnivMon).len(), 40); +} diff --git a/crates/types/src/summary_type/statistic.rs b/crates/types/src/summary_type/statistic/mod.rs similarity index 59% rename from crates/types/src/summary_type/statistic.rs rename to crates/types/src/summary_type/statistic/mod.rs index 3dc78475f..47a7e3896 100644 --- a/crates/types/src/summary_type/statistic.rs +++ b/crates/types/src/summary_type/statistic/mod.rs @@ -138,6 +138,25 @@ pub enum SketchStatistic { TopK { k: usize }, } +impl SketchStatistic { + /// The statistic a summary is read for to answer `intent`, or `None` + /// when no sketch statistic serves the intent. + pub fn for_intent(intent: &AggIntent) -> Option { + Some(match intent { + AggIntent::Quantile { q, .. } => Self::Quantile { q: *q }, + AggIntent::Cardinality { .. } => Self::Cardinality, + AggIntent::FrequencyL2 { .. } => Self::FrequencyL2, + AggIntent::FrequencyEntropy { .. } => Self::FrequencyEntropy, + AggIntent::TopK { k, .. } => Self::TopK { k: *k }, + AggIntent::Count => Self::PointCount { + key: ColumnRef::SampleValue, + value: None, + }, + _ => return None, + }) + } +} + /// The metric whose raw data samples, or plain sums of them, are every value /// `input` outputs, or `None`: walking down through a time range, an /// `offset`-only time shift (it moves the window, not the samples) and a @@ -168,136 +187,4 @@ pub fn raw_data_samples_of_metric(input: &I) -> Option } #[cfg(test)] -mod tests { - use super::*; - use crate::ir::operator::arguments::Reduction; - use crate::ir::schema::{DataType, Field, Schema}; - use crate::ir::{OperatorNode, TimeRangeKind}; - use std::rc::Rc; - use std::time::Duration; - - /// The raw data samples of `m` reach a node through a range, an - /// `offset`-only time shift and a plain `sum`; a rate ends them. - #[test] - fn raw_data_samples_of_metric_follows_ranges_shifts_and_plain_sums() { - let node = - |operator: Operator>| OperatorNode::new_shared(operator).unwrap(); - let rows = node(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })); - let window = node(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: rows, - })); - let per_series = |measure: AggIntent| { - node(Operator::NonASAP(NonASAPOp::Aggregate { - child: Rc::clone(&window), - reduction: Reduction::PerEntity, - measures: vec![measure], - output_names: vec![String::new()], - filters: vec![], - having: None, - })) - }; - assert_eq!(raw_data_samples_of_metric(&window), Some("m".to_string())); - let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { - shift: crate::ir::operator::arguments::TimeShift { - offset_ms: 60_000, - at: None, - }, - child: Rc::clone(&window), - })); - assert_eq!(raw_data_samples_of_metric(&shifted), Some("m".to_string())); - assert_eq!( - raw_data_samples_of_metric(&per_series(AggIntent::Sum { col: None })), - Some("m".to_string()) - ); - assert_eq!( - raw_data_samples_of_metric(&per_series(AggIntent::Rate)), - None - ); - } - - #[test] - fn keyed_summary_input_and_topk_evaluation_round_trip() { - for input in [ - SummaryUpdate { - item: Some(SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { excluding: vec![] }, - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("service".into()), - )])), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }, - ] { - let input_json = serde_json::to_string(&input).unwrap(); - assert_eq!( - serde_json::from_str::(&input_json).unwrap(), - input - ); - let query = SketchStatistic::TopK { k: 10 }; - let json = serde_json::to_string(&query).unwrap(); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - query - ); - } - } - - /// The raw-data-sample and summary-sample renames keep their JSON names, - /// so committed plans and workloads still read. - #[test] - fn renamed_sample_terms_keep_their_wire_names() { - use crate::ir::schema::FieldDataType; - use crate::summary_type::{SamplingKind, SamplingParams}; - let json = |value: serde_json::Value| value.to_string(); - let raw = WeightDomain::RawDataSamplesOfMetric { - metric: "m".to_string(), - }; - assert_eq!( - serde_json::to_string(&raw).unwrap(), - json(serde_json::json!({"kind": "samples_of_metric", "metric": "m"})) - ); - for (proof, name) in [ - ( - NonNegativeWeightProof::CounterRawDataSamples, - "counter_samples", - ), - ( - NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, - "declared_non_negative_samples", - ), - ] { - assert_eq!( - serde_json::to_string(&proof).unwrap(), - format!("\"{name}\"") - ); - } - let summary_sample = FieldDataType::SummarySample( - SamplingKind::Reservoir, - SamplingParams::Reservoir { size: 64 }, - ); - assert_eq!( - serde_json::to_string(&summary_sample).unwrap(), - json(serde_json::json!({"Sample": ["Reservoir", {"Reservoir": {"size": 64}}]})) - ); - } -} +mod tests; diff --git a/crates/types/src/summary_type/statistic/tests.rs b/crates/types/src/summary_type/statistic/tests.rs new file mode 100644 index 000000000..b485d6050 --- /dev/null +++ b/crates/types/src/summary_type/statistic/tests.rs @@ -0,0 +1,130 @@ +use super::*; +use crate::ir::operator::arguments::Reduction; +use crate::ir::schema::{DataType, Field, Schema}; +use crate::ir::{OperatorNode, TimeRangeKind}; +use std::rc::Rc; +use std::time::Duration; + +/// The raw data samples of `m` reach a node through a range, an +/// `offset`-only time shift and a plain `sum`; a rate ends them. +#[test] +fn raw_data_samples_of_metric_follows_ranges_shifts_and_plain_sums() { + let node = |operator: Operator>| OperatorNode::new_shared(operator).unwrap(); + let rows = node(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })); + let window = node(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: rows, + })); + let per_series = |measure: AggIntent| { + node(Operator::NonASAP(NonASAPOp::Aggregate { + child: Rc::clone(&window), + reduction: Reduction::PerEntity, + measures: vec![measure], + output_names: vec![String::new()], + filters: vec![], + having: None, + })) + }; + assert_eq!(raw_data_samples_of_metric(&window), Some("m".to_string())); + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: crate::ir::operator::arguments::TimeShift { + offset_ms: 60_000, + at: None, + }, + child: Rc::clone(&window), + })); + assert_eq!(raw_data_samples_of_metric(&shifted), Some("m".to_string())); + assert_eq!( + raw_data_samples_of_metric(&per_series(AggIntent::Sum { col: None })), + Some("m".to_string()) + ); + assert_eq!( + raw_data_samples_of_metric(&per_series(AggIntent::Rate)), + None + ); +} + +#[test] +fn keyed_summary_input_and_topk_evaluation_round_trip() { + for input in [ + SummaryUpdate { + item: Some(SummaryInputExpr::EntityIdentity( + EntityIdentity::PromqlLabelSet { excluding: vec![] }, + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + }, + SummaryUpdate { + item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( + ColumnRef::Named("service".into()), + )])), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain: WeightDomain::UnknownOrSigned, + }, + ] { + let input_json = serde_json::to_string(&input).unwrap(); + assert_eq!( + serde_json::from_str::(&input_json).unwrap(), + input + ); + let query = SketchStatistic::TopK { k: 10 }; + let json = serde_json::to_string(&query).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + query + ); + } +} + +/// The raw-data-sample and summary-sample renames keep their JSON names, +/// so committed plans and workloads still read. +#[test] +fn renamed_sample_terms_keep_their_wire_names() { + use crate::ir::schema::FieldDataType; + use crate::summary_type::{SamplingKind, SamplingParams}; + let json = |value: serde_json::Value| value.to_string(); + let raw = WeightDomain::RawDataSamplesOfMetric { + metric: "m".to_string(), + }; + assert_eq!( + serde_json::to_string(&raw).unwrap(), + json(serde_json::json!({"kind": "samples_of_metric", "metric": "m"})) + ); + for (proof, name) in [ + ( + NonNegativeWeightProof::CounterRawDataSamples, + "counter_samples", + ), + ( + NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, + "declared_non_negative_samples", + ), + ] { + assert_eq!( + serde_json::to_string(&proof).unwrap(), + format!("\"{name}\"") + ); + } + let summary_sample = FieldDataType::SummarySample( + SamplingKind::Reservoir, + SamplingParams::Reservoir { size: 64 }, + ); + assert_eq!( + serde_json::to_string(&summary_sample).unwrap(), + json(serde_json::json!({"Sample": ["Reservoir", {"Reservoir": {"size": 64}}]})) + ); +} diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support/forwarders.rs similarity index 99% rename from crates/types/src/test_support.rs rename to crates/types/src/test_support/forwarders.rs index d55290ee6..f5a9ae22d 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support/forwarders.rs @@ -4,8 +4,6 @@ //! to the crate-private function of the same name, except the fixture //! [`population_over_selector`], which the executor's tests build. -pub mod memo_fixtures; - use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; use crate::plan::materialized_view::RawDataWindow; @@ -76,7 +74,7 @@ pub fn physical_asap_dag_at_query_time( /// over a current-series population of the single selector under it, /// grouped by `grouping`, with no matchers. The caller states the statistic /// and grouping; nothing is recognized. No planner pass offers this sub-DAG -/// yet (#681). +/// yet. pub fn population_over_selector( root: &Rc, statistic: crate::ir::operator::maintained_population::PopulationStatistic, diff --git a/crates/types/src/test_support/memo_builders.rs b/crates/types/src/test_support/memo_builders.rs new file mode 100644 index 000000000..09c5765d6 --- /dev/null +++ b/crates/types/src/test_support/memo_builders.rs @@ -0,0 +1,251 @@ +//! Memo builders the memo's unit tests and the stage and accuracy crates' +//! tests share: each adds groups with the candidates Pass 1 would give +//! them (`Memo::find_or_add_group`, `Memo::add_candidate`) and queries +//! (`Memo::add_query`), so no test calls a stage crate. The expressions +//! come from [`memo_fixtures`](super::memo_fixtures). + +use crate::accuracy::AccuracyTarget; +use crate::ir::operator::arguments::Reduction; +use crate::ir::operator::AggIntent; +use crate::ir::schema::series_identity::with_promql_series_identity; +use crate::ir::schema::{DataType, FieldDataType}; +use crate::ir::{NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr}; +use crate::stages::{ + listed_build_parameters, CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, Memo, + MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, +}; +use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use crate::workload::{DataWorkload, SourceFacts}; + +use super::memo_fixtures::{range, readout, scan, state_grouped_by}; + +/// Adds `operator` with `parameters` to `group`, as Pass 1 does. +pub fn try_candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> Result { + memo.add_candidate( + group, + CandidateForMemoGroup::new( + operator, + parameters, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) +} + +/// [`try_candidate`], which must succeed. +pub fn candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> CandidateForMemoGroupId { + try_candidate(memo, group, operator, parameters).unwrap() +} + +/// The group of `operator`, with `operator` as its exact candidate. +pub fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + candidate(memo, group, operator, None); + group +} + +/// The [`group`] of a `TimeSeries(metric)` scan listing Utf8 `labels` +/// whose label matchers are `matchers`, its series identity realized as +/// the planner realizes it before Stage 0 (`with_promql_series_identity`): +/// a closed schema whose last column is `$promql_series_identity`, keyed by +/// it and the timestamp. +pub fn scan_group( + memo: &mut Memo, + metric: &str, + labels: &[&str], + matchers: Vec>, +) -> MemoGroupId { + let labels: Vec<(&str, DataType)> = labels.iter().map(|l| (*l, DataType::Utf8)).collect(); + let mut operator = scan(metric, &labels, matchers); + let open = + OperatorNode::new_shared(operator.map_children(|_| unreachable!("a scan has no input"))) + .unwrap(); + let realized = with_promql_series_identity(&open).unwrap().schema.clone(); + if let Operator::NonASAP(NonASAPOp::Scan { schema, .. }) = &mut operator { + *schema = realized; + } + group(memo, operator) +} + +/// Adds the query rooted at group `root` with `target` (`None`: accuracy +/// not enforced), returning one value per evaluation. +pub fn query(memo: &mut Memo, root: MemoGroupId, target: Option) -> usize { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(root), + accuracy_target: target, + number_of_values_per_query_evaluation: Some(1), + }) +} + +/// Candidate 0 of `group`: the exact candidate [`group`] adds first. +pub fn own_candidate(group: MemoGroupId) -> CandidateForMemoGroupId { + CandidateForMemoGroupId { + group, + index_in_group: 0, + } +} + +/// A data workload declaring only that `metric` holds `series_count` +/// series. +pub fn with_series(metric: &str, series_count: u64) -> DataWorkload { + DataWorkload { + sources: [( + metric.to_string(), + SourceFacts { + series_count: Some(series_count), + ..SourceFacts::default() + }, + )] + .into(), + ..DataWorkload::default() + } +} + +/// The state group of a sketch build of `algorithm` over `child` grouped +/// by `reduction`, one build candidate per listed θ +/// (`listed_build_parameters`). +pub fn sketch_state( + memo: &mut Memo, + child: MemoGroupId, + algorithm: SketchAlgorithm, + reduction: Reduction, +) -> MemoGroupId { + let operator = state_grouped_by(FieldDataType::Sketch(algorithm), reduction, child); + let state = memo.find_or_add_group(operator.clone()).unwrap(); + for parameters in listed_build_parameters(&operator) { + candidate(memo, state, operator.clone(), parameters); + } + state +} + +/// The [`group`] of a readout of `statistic` over `state`. +pub fn estimate(memo: &mut Memo, state: MemoGroupId, statistic: SketchStatistic) -> MemoGroupId { + group(memo, readout(statistic, state)) +} + +/// `quantile_over_time(q, m[seconds])` per series of `m` by `job` through +/// a sketch state of `algorithm`: the range group, the state group and the +/// readout group. +pub fn quantile_over_time( + memo: &mut Memo, + algorithm: SketchAlgorithm, + q: f64, + seconds: u64, +) -> (MemoGroupId, MemoGroupId, MemoGroupId) { + let input = scan_group(memo, "m", &["job"], vec![]); + let window = group(memo, range(seconds as i64 * 1000, input)); + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let readout = estimate(memo, state, SketchStatistic::Quantile { q }); + (window, state, readout) +} + +/// The readouts Pass 1 offers one quantile group, per summary type. +pub struct QuantileReadouts { + /// The `Aggregate` group `quantile_over_time(q, window)` per series. + pub result: MemoGroupId, + /// One per algorithm. + pub by_algorithm: Vec, +} + +/// One summary type's readout of a quantile group ([`QuantileReadouts`]). +pub struct QuantileReadout { + pub algorithm: SketchAlgorithm, + /// The state group, one build per listed θ. + pub state: MemoGroupId, + /// The candidate of the quantile group reading the state: the readout + /// itself, or a renaming `Project` over the readout group. + pub candidate_of_result: CandidateForMemoGroupId, + /// The readout (`SummaryEstimate`) candidate. + pub readout: CandidateForMemoGroupId, +} + +/// `quantile_over_time(q, window)` per series as Pass 1 leaves it: the +/// aggregate group with its exact candidate and, per algorithm, a state +/// group (one build per listed θ) and a candidate of the aggregate group +/// reading it (through a renaming `Project` over a readout group when the +/// column names differ, as Pass 1 does). +pub fn quantile_over_time_readouts( + memo: &mut Memo, + window: MemoGroupId, + q: f64, + algorithms: &[SketchAlgorithm], +) -> QuantileReadouts { + let result = group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: window, + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Quantile { col: None, q }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ); + let mut by_algorithm = Vec::new(); + for &algorithm in algorithms { + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let direct = readout(SketchStatistic::Quantile { q }, state); + let (candidate_of_result, readout) = match try_candidate(memo, result, direct, None) { + Ok(id) => (id, id), + Err(CandidateError::OutputDiffersFromGroup { .. }) => { + let readout = estimate(memo, state, SketchStatistic::Quantile { q }); + let to = &memo.groups()[result.0 as usize].group_properties.schema; + let renamed = Operator::NonASAP(NonASAPOp::Project { + cols: to + .fields + .iter() + .enumerate() + .map(|(column, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(column), + }) + .collect(), + child: readout, + }); + ( + candidate(memo, result, renamed, None), + own_candidate(readout), + ) + } + Err(error) => panic!("{error}"), + }; + by_algorithm.push(QuantileReadout { + algorithm, + state, + candidate_of_result, + readout, + }); + } + QuantileReadouts { + result, + by_algorithm, + } +} + +/// The candidates of `group`. +pub fn candidates_of(memo: &Memo, group: MemoGroupId) -> &[CandidateForMemoGroup] { + &memo.groups()[group.0 as usize].candidates +} + +/// The groups whose expression matches `pattern`. +pub fn groups_where( + memo: &Memo, + pattern: impl Fn(&Operator) -> bool, +) -> Vec { + memo.groups() + .iter() + .enumerate() + .filter(|(_, group)| pattern(&group.expression)) + .map(|(index, _)| MemoGroupId(index as u32)) + .collect() +} diff --git a/crates/types/src/test_support/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs index 4e494efa0..9a23b9a4e 100644 --- a/crates/types/src/test_support/memo_fixtures.rs +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -1,13 +1,21 @@ -//! Memo expressions over group ids that the memo's unit tests and this +//! Memo expressions over group ids that the memo's unit tests, this //! crate's memo integration tests (`memo_group_properties`, -//! `memo_aggregate_coverage`) build their examples from. +//! `memo_aggregate_coverage`) and the stage crates' tests build their +//! examples from. The builders that add them to a memo are in +//! [`memo_builders`](super::memo_builders). use std::time::Duration; -use crate::ir::operator::arguments::{AtModifier, Reduction, Source, SourceColumn, TimeShift}; +use crate::ir::operator::arguments::{ + AtModifier, BinaryOpKind, Reduction, Source, SourceColumn, TimeShift, +}; use crate::ir::operator::AggIntent; -use crate::ir::scalar::ColumnRef; +use crate::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind}; +use crate::ir::scalar::{ColumnRef, ScalarValue}; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; -use crate::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; +use crate::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, + TimeRangeKind, +}; use crate::stages::MemoGroupId; use crate::summary_type::{SketchStatistic, SummaryUpdate}; @@ -85,11 +93,20 @@ pub fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Oper /// `SummaryAgg[summary_type]` of the sample value per series. pub fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { + state_grouped_by(summary_type, Reduction::PerEntity, child) +} + +/// `SummaryAgg[summary_type]` of the sample value, grouped by `reduction`. +pub fn state_grouped_by( + summary_type: FieldDataType, + reduction: Reduction, + child: MemoGroupId, +) -> Operator { Operator::ASAP(ASAPOp::SummaryAgg { child, summary_type, input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, + reduction, filter: None, }) } @@ -133,3 +150,38 @@ pub fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator literal`, with `semantics` (the frontend's comparison +/// rules). +pub fn compare( + column: usize, + op: CompareOpKind, + literal: ScalarValue, + semantics: ExprSemantics, +) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(literal)), + semantics, + } +} + +/// `lhs rhs`, value by value, without vector matching. +pub fn arithmetic( + kind: ArithmeticOpKind, + lhs: MemoGroupId, + rhs: MemoGroupId, +) -> Operator { + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(kind), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) +} diff --git a/crates/types/src/test_support/mod.rs b/crates/types/src/test_support/mod.rs new file mode 100644 index 000000000..c6343f5d7 --- /dev/null +++ b/crates/types/src/test_support/mod.rs @@ -0,0 +1,11 @@ +//! Helpers that only tests call (this crate's integration tests and other +//! crates' tests), exposed under the `test-support` feature so the normal +//! public API stays as narrow as its production users need. + +/// Forwards to this crate's private functions, plus fixtures the executor's +/// tests build. +pub mod forwarders; +/// Memo builders the stage and accuracy crates' tests share. +pub mod memo_builders; +/// Operator expressions the memo builders and tests share. +pub mod memo_fixtures; diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index f7ea803bd..42d16a762 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -142,10 +142,12 @@ pub struct SourceFacts { /// it must hold for the longest evaluation window any of them uses; a /// filter or a shorter window only lowers it. Each group's own items /// count separately: the bound is per output row. The deployment must - /// enforce it; an observed distinct count is not a bound. Plan - /// selection binds it to give classic HLL (and a distinct-count UnivMon) - /// its failure probability per output row; without it that probability - /// is unknown. KMV's guarantee needs no bound. + /// enforce it; an observed distinct count is not a bound. It is for a + /// distinct-count guarantee whose failure probability per output row + /// depends on the number of distinct items (classic HLL, a + /// distinct-count UnivMon); no guarantee in `asap_accuracy` reads it + /// yet, so those readouts serve no query with an accuracy target. KMV's + /// guarantee needs no bound. #[serde(default)] pub max_distinct_per_evaluation: Option, /// How many distinct values each label (PromQL) or column (SQL) of the diff --git a/crates/types/src/workload/parsed_workload.rs b/crates/types/src/workload/parsed_workload.rs index c555562dc..af6328469 100644 --- a/crates/types/src/workload/parsed_workload.rs +++ b/crates/types/src/workload/parsed_workload.rs @@ -2,7 +2,7 @@ //! to the operator IR. //! //! This is the boundary between the query-language frontends and Stage 1, -//! logical ASAP-aware optimization (issues #429, #430). Everything downstream of lowering consumes this type +//! logical ASAP-aware optimization. Everything downstream of lowering consumes this type //! and never sees query text, a catalog, or a query language; everything the //! optimizer still needs about *demand* — recurrence, predictability, //! execution time, accuracy requirement — is read off the retained diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs index 94bc495db..7f1f4caba 100644 --- a/crates/types/tests/column_lineage.rs +++ b/crates/types/tests/column_lineage.rs @@ -19,8 +19,8 @@ use asap_types::ir::{ ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, }; use asap_types::stages::{ - CandidateExpression, CandidateForMemoGroup, ColumnLineage, LogicalProperties, - LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, ValueSign, + CandidateForMemoGroup, ColumnLineage, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, + ValueSign, }; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{ @@ -150,14 +150,8 @@ fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { ], child: g5, }); - let expression = CandidateExpression { operator }; - let candidate = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; + let candidate = + CandidateForMemoGroup::new(operator, None, OriginOfCandidateForMemoGroup::Pass1, vec![]); let id = memo.add_candidate(g3, candidate).unwrap(); assert_eq!(id.group, g3); } diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index ef8599935..a5afa977b 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -5,7 +5,7 @@ mod column_lineage; mod memo_aggregate_coverage; mod memo_group_properties; -mod physical_export; +mod physical_asap_dag; mod planner_vocabulary; mod schema_rebuilding; mod structure_contract; diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 5c96ed40b..5a2b74d2d 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -18,12 +18,14 @@ use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarVal use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; +use asap_types::plan::view_coverage; use asap_types::stages::{ - view_coverage, AggregateCoverage, CandidateExpression, CandidateForMemoGroup, ColumnLineage, - CoverageError, LogicalProperties, LogicalRequirements, Memo, MemoGroupError, MemoGroupId, - OriginOfCandidateForMemoGroup, + AggregateCoverage, CandidateForMemoGroup, ColumnLineage, CoverageError, Memo, MemoGroupError, + MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::{ + ExactKind, MergeInputRule, SketchAlgorithm, SketchParams, SummaryUpdate, }; -use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{filter, range, scan, series_column, shift, state}; use asap_types::workload::DataWorkload; @@ -350,7 +352,7 @@ fn scan_predicates_lift_and_their_states_merge() { /// G1 Scan latency /// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) /// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) -/// G4 candidate: SummaryMerge(G7) +/// G4 candidate: SummaryMerge[k=269](G7) /// ``` #[test] fn a_merge_over_a_chains_windows_covers_the_direct_build() { @@ -370,16 +372,12 @@ fn a_merge_over_a_chains_windows_covers_the_direct_build() { assert_eq!(time_of(&memo, g7), five_minutes_an_hour_ago); assert_eq!(coverage(&memo, g7), coverage(&memo, g4)); - let expression = CandidateExpression { - operator: merge(vec![g7]), - }; - let merge_of_windows = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; + let merge_of_windows = CandidateForMemoGroup::new( + merge(vec![g7]), + Some(SketchParams::Kll { k: 269 }), + OriginOfCandidateForMemoGroup::Pass1, + vec![], + ); memo.add_candidate(g4, merge_of_windows).unwrap(); let merged = memo.find_or_add_group(merge(vec![g7])).unwrap(); assert_eq!(coverage(&memo, merged), coverage(&memo, g4)); diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_asap_dag.rs similarity index 93% rename from crates/types/tests/physical_export.rs rename to crates/types/tests/physical_asap_dag.rs index 37acefdae..b9e388758 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_asap_dag.rs @@ -50,9 +50,10 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { .into_iter() .map(QueryRoot::Operator) .collect(); - let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) - .map(|(dag, _)| dag) - .unwrap(); + let dag = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time_with_node_ids(&roots) + .map(|(dag, _)| dag) + .unwrap(); dag.validate().unwrap(); assert_eq!(dag.roots.len(), 2); assert_eq!(dag.nodes.len(), 4, "scan and summary are exported once"); @@ -71,12 +72,13 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { right: Box::new(ScalarExpr::literal_f64(1.0)), semantics: ExprSemantics::Promql, }; - let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[ - QueryRoot::Scalar(scalar), - QueryRoot::Operator(Rc::clone(&read)), - ]) - .map(|(dag, _)| dag) - .unwrap(); + let dag = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time_with_node_ids(&[ + QueryRoot::Scalar(scalar), + QueryRoot::Operator(Rc::clone(&read)), + ]) + .map(|(dag, _)| dag) + .unwrap(); assert_eq!(dag.nodes.len(), 3, "the read plan is exported once"); assert_eq!(dag.roots.len(), 1); assert_eq!(dag.scalar_roots.len(), 1); @@ -178,7 +180,8 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { child: series("a"), })) .unwrap(); - let dag = asap_types::test_support::physical_asap_dag_at_query_time(&project).unwrap(); + let dag = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&project).unwrap(); dag.validate().unwrap(); let roles: Vec<_> = dag .edges @@ -214,7 +217,8 @@ fn swapped_edge_roles_are_rejected() { right: scan("b"), })) .unwrap(); - let mut dag = asap_types::test_support::physical_asap_dag_at_query_time(&join).unwrap(); + let mut dag = + asap_types::test_support::forwarders::physical_asap_dag_at_query_time(&join).unwrap(); dag.validate().unwrap(); for edge in &mut dag.edges { edge.role = match edge.role { @@ -235,7 +239,7 @@ fn swapped_edge_roles_are_rejected() { #[test] fn an_empty_batch_is_an_error() { assert!(matches!( - asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[]) + asap_types::test_support::forwarders::physical_asap_dag_at_query_time_with_node_ids(&[]) .map(|(dag, _)| dag), Err(PhysicalASAPDAGValidationError::NoRoots) )); @@ -266,7 +270,10 @@ fn a_raw_data_window_ends_its_offset_before_the_evaluation() { 600_000, ); assert_eq!(window.bounds_ms(3_600_000), (2_700_000, 3_300_000)); - assert_eq!(asap_types::test_support::lookback_ms(&window), 900_000); + assert_eq!( + asap_types::test_support::forwarders::lookback_ms(&window), + 900_000 + ); } /// High-fan-in merge/concat validation is independent of wire edge order and @@ -294,7 +301,6 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { lifetime: Lifetime::None, parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -310,7 +316,6 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { lifetime: Lifetime::None, parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 95e171a0a..5a61cb41b 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,4 +1,3 @@ -use asap_types::accuracy::AccuracyGuarantee; use asap_types::ir::operator::{AggIntent, Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -51,21 +50,20 @@ fn rebuilding_rederives_schema_for_both_categories() { let rebuilt = Rc::new(original.with_new_children(|_| replacement.clone()).unwrap()); assert_eq!( rebuilt.schema, - asap_types::test_support::output_schema(&rebuilt.operator).unwrap() + asap_types::test_support::forwarders::output_schema(&rebuilt.operator).unwrap() ); rebuilt.validate_structure().unwrap(); } } -/// Naming overrides survive rewrites without freezing types or assessed properties. +/// Naming overrides survive rewrites without freezing types. #[test] fn rebuilding_preserves_only_explicit_naming_overrides() { for asap in [false, true] { let original = aggregate(scan(DataType::Int64, "key"), asap); let mut schema = original.schema.clone(); schema.fields[0].name = "alias".into(); - let renamed = OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); + let renamed = OperatorNode::with_schema(original.operator.clone(), schema); let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); @@ -75,7 +73,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { rebuilt.schema.fields[0].plain_dtype(), Some(&DataType::Utf8) ); - assert!(rebuilt.guarantee.is_none()); rebuilt.validate_structure().unwrap(); } } @@ -275,10 +272,14 @@ fn a_current_series_population_matches_only_its_selectors_lookback() { lookback_ms: 1_000, }; let selector = series_selector(); - assert!(asap_types::test_support::matches_node(&spec, &selector)); + assert!(asap_types::test_support::forwarders::matches_node( + &spec, &selector + )); let wrong = CurrentSeriesInput { lookback_ms: 300_000, ..spec }; - assert!(!asap_types::test_support::matches_node(&wrong, &selector)); + assert!(!asap_types::test_support::forwarders::matches_node( + &wrong, &selector + )); } diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 16e76b000..a0e9200c3 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -4,7 +4,7 @@ use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ ExprSemantics, NonASAPOp, OperatorNode, OperatorResultKind, Predicate, ScalarExpr, }; -use asap_types::test_support::validate_asap_inputs; +use asap_types::test_support::forwarders::validate_asap_inputs; use std::rc::Rc; fn scan() -> Rc { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -28,7 +28,7 @@ fn invalid_predicates_are_rejected() { child: scan(), pred: Predicate(expr), }; - assert!(asap_types::test_support::validate_non_asap_inputs(&op).is_err()); + assert!(asap_types::test_support::forwarders::validate_non_asap_inputs(&op).is_err()); } } /// Common metadata must agree with the actual operation, including result kind. @@ -50,7 +50,7 @@ fn values_contract_is_checked() { rows: vec![row], schema: scan().schema.clone(), }; - assert!(asap_types::test_support::validate_non_asap_inputs(&op).is_err()); + assert!(asap_types::test_support::forwarders::validate_non_asap_inputs(&op).is_err()); } } /// Scalar typing validates every branch and never assigns placeholder types. @@ -152,7 +152,6 @@ fn shared_construction_derives_both_operator_categories() { ); assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); - assert!(state.guarantee.is_none()); state.validate_structure().unwrap(); assert!( diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index c70d13489..ecef365b2 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -72,7 +72,7 @@ fn a_chain_reads_an_instant_vector() { assert!(empty.validate_structure().is_err()); } -/// JSON written before #1163 names the chain `PaneChain` and its width +/// JSON written before the tumbling-window rename names the chain `PaneChain` and its width /// `pane_width_ms`; it reads as today's operator. #[test] fn names_before_the_tumbling_window_rename_still_read() { diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index 74bd32816..3cc1e1c9c 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -20,7 +20,7 @@ not turn an unknown guarantee into a satisfied accuracy requirement. Here, **certified for a target** means Planner has a fully known `AccuracyGuarantee` for the candidate whose evaluated error is within the -target's (`asap_accuracy::target_limit`). **Uncertified** means Planner cannot make that claim: +target's (`asap_accuracy::within_limit`). **Uncertified** means Planner cannot make that claim: the guarantee is absent, contains unknown terms, or is known not to meet the target. An uncertified summary may still be a well-formed logical candidate; this label says nothing about whether the backend can physically execute it. @@ -29,7 +29,7 @@ The exact path (the pre-ASAP sub-DAG kept by `retain_exact`) has an exact guaran | State | Planner representation | Consequence / next step | |---|---|---| | Known guarantee | `AccuracyGuarantee` with evaluable bound and failure probability | Planner checks whether the guarantee satisfies the query's accuracy target. If this candidate is selected, the backend checks whether its implementation can realize the selected summary; it does not re-decide the accuracy target. | -| Missing accuracy/domain evidence | Symbolic `BoundExpr::Unknown` or `ProbabilityExpr::Unknown`, or `guarantee: None` on a constructible summary | Inspect `ReplacementSubDAG::has_missing_accuracy_evidence()`, obtain applicable evidence or apply explicit policy; do not claim certification. | +| Missing accuracy/domain evidence | An unknown `ProbabilityExpr::Unknown`, or `guarantee: None` on a constructible summary | Inspect `ReplacementSubDAG::has_missing_accuracy_evidence()`, obtain applicable evidence or apply explicit policy; do not claim certification. | | Missing cost | `CostModel::candidate_cost()` returns `None` for a `ReplacementSubDAG` (including a non-finite or negative legacy estimate) | Keep that logical summary/rewrite candidate in `CandidateLogicalASAPDAGs` for inspection; provide a comparable cost before selecting it by cost. This does not make it a deployable physical plan. | | Unknown runtime support | `ReplacementSubDAG::runtime_support_evidence(model)` returns `None` | Candidate remains visible; bind a concrete implementation and confirm support before deployment. | | Known invalid evidence or impossible semantics | No candidate; where supported, a `RejectedCandidate` records the error | Do not deploy. | diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index b1f7c8b2e..d546355fe 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -9,7 +9,7 @@ of its inputs. A locally accurate sketch does not automatically make an entire query accurate. Planner owns the interpretation of accuracy targets, estimator models, error -propagation, budget allocation and candidate legality. Deployments provide +propagation, readout requirements and candidate legality. Deployments provide source contracts and evidence with a defensible scope. Cost models compare resource costs; runtime implementations execute the selected semantics and must honor the conditions attached to them. @@ -26,7 +26,7 @@ These three concepts have different roles: | Concept | Meaning | Owner | |---|---|---| | Accuracy target | What the caller requires of the result | Query requirements and aggregate intent | -| Result guarantee | What a particular computation can establish | Estimator and composition models in Planner | +| Result guarantee | What a particular computation can establish | Estimator models and error propagation in Planner | | Evidence | Facts or trusted contracts needed to establish that guarantee | Source or deployment evidence provider | `AccuracyTarget` is the authoritative requirement type: @@ -65,25 +65,32 @@ share meets a target over a larger share but not a per-value target, and only a per-value guarantee is restated over a scope. A scope bounds only the failure probability; the error bound is unchanged. -Every estimator and composition produces a per-value guarantee (for a grouped +Every estimator produces a per-value guarantee (for a grouped query, one per group: a Kmv or HLL distinct count per group, a KLL or DDSketch quantile per group, a Count-Min or Count Sketch frequency per key, a Hydra estimate per subpopulation; an exact value fails with probability 0). Stated over a scope, a per-value failure probability `δ` becomes `V · δ` by the union bound, without assuming independence, so a sketch shared by several groups (Hydra, a keyed Count-Min) is covered the same way -(`AccuracyGuarantee::over_scope`). A guarantee whose own failure probability is -unknown (generic HLL) stays unmet under every scope with a δ, and top-k -membership needs its own evidence under every scope. - -Pass 1 sizes each approximate input of a candidate with `n` accuracy-budget -parts for `δ / (n · V)`, so the union bound gives δ over the covered values. -A candidate several queries reach is sized for the smallest `δ / (n · V)` -among them, and a build Pass 2 shares among several readers takes the -strictest reader's size. Stage 3 only checks the composed guarantee; it never -resizes. At equal δ and `n`, the scope covering the most values is the -strictest: every group (`G`, when `G` is above 100), then a 1% share (100), -then a 5% share (20), then a single value (1). A whole-result target with an unbound `G` +(`AccuracyScope::max_values_covered` gives `V`). A guarantee whose own failure probability is +unknown (generic HLL) serves no query with an accuracy target +(`RootPruneReason::NoGuarantee`), and a top-k readout is admitted on its +heap size and its value guarantee alone (the contracts table). + +Pass 2 gives each sketch readout the error and per-value failure probability +`δ / max(V, R)` it must meet, with `R` the readout group's values per +evaluation (the series its scans read): a `max` over 1000 readouts returns +one value yet fails when any readout does, so the union bound gives δ over +every readout read; +Pass 1's candidate takes the first requirement level and Pass 2 adds a copy +per further level. A readout several queries reach gets the union of their +levels. Pass 2 sizes nothing: a state has one build per listed θ, and each +edge checks the guarantee of its producer's θ at the required failure +probability, so a build several readers share serves the readers whose +levels its θ meets. Stage 3 only checks the guarantee; it never resizes. At equal +δ, the scope covering the most values is the strictest: every group (`G`, +when `G` is above 100), then a 1% share (100), then a 5% share (20), then a +single value (1). A whole-result target with an unbound `G` enlarges nothing and stays unmet. A grouped result's `G` is bound by the distinct values its sources declare for its group keys (`SourceFacts.distinct_values_per_label`, and @@ -94,11 +101,11 @@ most the series read. With each of 625 endpoints in one of 25 services and most 625 values, not 25 x 625. A key counts only when it is a source column passed through unchanged; a relabeled or computed key leaves `G` unbound. So a sketch meets a whole-result target over SQL groups when every grouping column's distinct values are declared, and over 25 services -and 25 endpoints sizes each group's sketch for `δ / 625`. Nothing declared +and 25 endpoints requires each group's sketch to meet `δ / 625`. Nothing declared leaves `G` unbound; no default stands in for it. -An `AccuracyGuarantee` contains an error metric, an error-bound expression, a -failure-probability expression and provenance. The metric is essential: +An `AccuracyGuarantee` contains an error metric, an error bound and a +failure probability (possibly unknown, as HLL's). The metric is essential: | Metric | Quantity bounded | |---|---| @@ -116,8 +123,8 @@ rule and any evidence that rule needs. Bounds and probabilities can remain symbolic. An unknown population size, normalization factor or failure probability stays unknown; it is not replaced -by zero. Provenance records estimator parameters and contracts, child -guarantees, composition steps, allocated targets and unavailable evidence. +by zero. Provenance records that a value is exact, or the sketch evaluation with its +estimator parameters and contract. This makes an accuracy decision explainable without reconstructing it from query text or cost estimates. @@ -125,16 +132,16 @@ query text or cost estimates. ```mermaid flowchart TD - Request[Query semantics and accuracy target] --> Generate[Generate candidates and size parameters] + Request[Query semantics and accuracy target] --> Generate[Generate candidates with listed parameters] Evidence[Scoped source contracts and evidence] --> Generate Generate --> Local[Derive local evaluation guarantees] Evidence --> Local - Local --> Compose[Propagate guarantees through the DAG] + Local --> Compose[Propagate required errors through the DAG] Evidence --> Compose Compose --> Check[Check the composed target] Check -->|Satisfied with sufficient evidence| Eligible[Eligible for cost comparison] Check -->|Evidence missing| Pending[Retain an uncertified candidate for explanation] - Check -->|Known violation or unsupported composition| Reject[Reject with a reason] + Check -->|Known violation or unsupported propagation| Reject[Reject with a reason] Eligible --> Select[Select compatible candidates across the workload] Select --> Bind[Bind implementations and deployment resources] ``` @@ -156,26 +163,26 @@ missing-evidence gate still matters during selection. A successful accuracy check also does not prove that a runtime implements the plan, that stored state is ready, or that a complete deployment cost is available. -## Local estimator models and parameter sizing +## Local estimator models and parameter lists A local model describes a specific evaluation of a specific estimator with committed parameters and applicable assumptions. A family name or a parameter such as HLL precision is not, by itself, a confidence certificate. -Sizing proposes parameters for a local target. Planner then derives the -actual guarantee from those parameters and checks it. Parameter rounding, -implementation limits, or conservative probability bounds can make a proposed -configuration insufficient; sizing must not bypass that check. +Planner does not size parameters for a target. Each algorithm has a listed +set of parameters θ; Planner derives the guarantee of each listed θ at the +required failure probability and checks it against the requirement. +Implementation limits or conservative probability bounds can make a listed θ +insufficient for a requirement; that θ then serves no reader that needs more. The built-in models currently include: | Model | Current accuracy contract and limits | |---|---| -| Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require propagation | +| Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require error propagation | | KLL | Normalized rank error from the empirical 99% calibration `2.296 / k^0.9723`, fitted on Apache DataSketches' C++ KLL (`a9b42755072b`); below a failure probability of 0.01 that error is scaled by the Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`, a model rather than a measurement, and the guarantee names that contract apart | -| DDSketch | Relative value error from alpha under the supported estimator/domain contract; sensitive compositions require additional domain evidence | +| DDSketch | Relative value error from alpha under the supported estimator/domain contract; operators sensitive to sign or range require additional domain evidence | | Generic HLL | RSE magnitude with unknown failure probability; not a general confidence theorem | -| Bounded Classic HLL | Source-conditioned relative-error/failure-probability bound and precision sizing for the linear-counting branch | | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | | KMV | Provable distinct-count bound from the `k` smallest hashes | @@ -188,82 +195,132 @@ another implementation with the same algorithm name. In particular, a named empirical calibration is different from an arbitrary benchmark's maximum observed error; both its confidence and applicability must remain explicit. -Sizing is each algorithm's predefined parameter list, `parameter_list` -(`crates/types/src/summary_type/parameter_lists.rs`), which a test keeps equal -to the estimator modules' closed-form sizing over an error grid; a summary shared by several readers is sized with each -parameter's largest value among them. Where an error bound grows as the failure -probability shrinks (a KLL's `k`, a distinct-count UnivMon's heap), each -reader's parameter is first resized to the readers' smallest failure -probability at no larger error, so the shared summary meets every reader at -the one failure probability it states. Accuracy validation is independent of -sizing: Stage 3 checks the guarantee derived from the committed parameters. -The design boundary is that sizing never grants accuracy authority to the -cost model. +The θ lists are each algorithm's predefined parameter list, `parameter_list` +(`crates/types/src/summary_type/parameter_lists.rs`), weakest first. Pass 1 +and Pass 2 add one build per listed θ, and the logical-edge check keeps an +edge only when the guarantee of the producer's θ, at the readout's required +failure probability, is within the required error. Where an error bound grows +as the failure probability shrinks (a KLL's `k`), the same θ meets a reader +at 1e-2 and misses one at 1e-6; no θ is resized for a reader. Accuracy +validation is independent of cost: the guarantee is derived from the +committed parameters, and the cost model never grants accuracy authority. -## Composing guarantees through a DAG +## Propagating errors through a DAG The accuracy crate supplies theoretical guarantees (`theoretical_guarantee`), -propagation rules and the comparison limit of a target (`target_limit`); the -deployment's measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a -theoretical guarantee. The rules are conservative: an unregistered -composition over approximate inputs is rejected rather than treated as exact. - -The principal composition rules are: - -| Computation | Accuracy reasoning | +each operator's error contract (`ErrorContract`, gathered per memo state in +`ErrorMetricsOfMemo`) and the comparison of a bound with a target +(`within_limit`); the deployment's measurements +(`AccuracyEvidence::measured_guarantee`) take precedence over a theoretical +guarantee. Only a sketch readout introduces error. Every other operator +passes its inputs' errors on through a contract, and the contracts are +conservative: an operator with no contract for a metric fails the edge of an +input whose error is in that metric (`RootPruneReason::MetricRule`), so an +approximate value is never treated as exact. + +A contract is a linear bound: the consumer's output error is at most +`Σ β_i · ε_i` over its inputs' errors `ε_i`, valid while each input stays +within its validity cap `b_i` (`ErrorContract::error_factor_per_input` and +`linear_bound_holds_up_to_per_input`). The principal contracts are: + +| Operator | Contract | |---|---| -| Supported computation over exact inputs | Only its own local estimator error remains; a supported exact operation remains exact | -| Additive absolute-error composition | Add compatible error bounds and union-bound their failure events | -| Relative-error composition | Include the multiplicative cross term, subject to the required sign/domain conditions | -| Registered Lipschitz transform | Scale the input bound by the registered constant and include local error | -| Exact sum over approximate values | Convert compatible errors to absolute units and account for input multiplicity; missing scale/count evidence remains unknown | -| Exact average or extremum over approximate values | Use the registered absolute-error bound and account for failures across the input population | -| Division | Two relative errors (or an exact numerator over a relative error) compose in closed form, `(a + b)/(1 − b)` for `b < 1`, with no domain evidence; any other approximate denominator needs enforced operand domains bounded away from zero; rank-error division is unsupported | -| Counter rate/increase over approximate samples | No general distribution-free rule for reset detection and extrapolation; exact inputs remain a distinct supported case | -| Candidate-based TopK | Require membership/completeness evidence; frequency accuracy and exact reranking alone are insufficient | - -Probability composition uses union bounds, without assuming independence -between child errors. Estimator-local assumptions, such as independent bucket -hashing, must be stated separately. Shared DAG nodes do not create independent -errors merely because several consumers reference them, and sharing does not -justify reducing the failure budget. Per-result guarantees also do not imply -a simultaneous guarantee across every query or evaluation time in a dashboard. - -For relative errors, two eligible layers with errors `e1` and `e2` compose as -`(1 + e1)(1 + e2) - 1`, not simply `e1 + e2`. A ratio of relative errors `a` -over `b` is within `max((1 + a)/(1 − b) − 1, 1 − (1 − a)/(1 + b)) = (a + b)/(1 − b)`. -A relative bound below 1 keeps zero at zero and keeps the sign, so a zero -denominator gives the same non-finite answer exactly and approximately. The -rule relies on each operand's own relative guarantee (for DDSketch, its -estimator contract: one-signed values within the mapping's range). A ratio -of operands whose relative bound is still symbolic in Stage 1 (e.g. a sum of -quantiles gated on the values' sign) gets no guarantee. A sum or difference of relative -errors has no relative bound (cancellation), and a rank error (KLL) has no -value bound, so neither composes through division. Likewise, an exact sum above -approximate values retains their uncertainty: exact arithmetic does not -recover information lost below it. - -## Allocating an end-to-end budget - -`AccuracyBudgetAllocator` proposes local targets for a composition. These -allocations create candidates; they are not proofs that the candidates work. -Every proposed composition is checked again using the actual guarantees. - -The default equal-split allocator divides an additive error budget across the -approximate layers. For relative error, it uses local error -`1 - (1 + epsilon)^(-1/n)` for `n` layers. The true error of any mix of -products and quotients stays within the target (each layer contributes at most -`1/(1 - e)`); the composed bound meets it for one product or quotient of two -layers, not for a nested quotient or an exact numerator over a lone -approximate denominator. When the target includes delta, it -splits that budget across the layers for union-bound composition. An exact -target does not receive an approximate allocation. - -For example, under a supported two-layer relative-error composition, a 10% -end-to-end budget gives each layer approximately 4.65%, not 5%. If the failure -budget is 1%, each layer receives 0.5%. A model with only a fixed 1% failure -contract cannot automatically satisfy that allocation; another supported -configuration or candidate is needed. +| Sort, TimeRange, TimeShift, Concat, a readout, a finalize, a summary merge | `β = 1` in any metric | +| Filter on columns copied from a source (labels and keys) | `β = 1` in any metric; a filter comparing a computed value has none | +| Project | Judged on the items reading a computed value (labels, the time and the series identity are copied): `β = 1` in any metric for a copied column; absolute error, `β = 1`, for `abs`, `clamp`, `clamp_min`, `clamp_max` (also PromQL's `promql_` functions) and a negation; relative error, `β = 1`, for a product with or quotient by a nonzero constant | +| Aggregate or exact-accumulator build of counts, min, max, avg | Counts `β = 0`; min and max absolute or relative with `β = 1`; avg absolute with `β = 1`; only when every group-by key is copied from a source (a `by` key as listed; for `without`, every column but the time, the sample value and the excluded ones); none with measure filters or `HAVING`, or with a key reading an estimate | +| `+`, `-` | Absolute error, `β = 1` | +| `/` | Relative error, `β = 1/(1 − b)` on both operands, the denominator capped at `b` | +| `*` | Relative error, `β = (1 + b, 1)`, the right operand capped at `b` | +| Sum, comparison, ranking, counter rate or increase, any other function | No contract: the edge of an approximate input fails | +| Top-k over a sketch | Checked: the heap size is at least `k`, and the readout's value guarantee meets the requirement; membership has no check of its own | + +`b` is `RELATIVE_OPERAND_VALIDITY_CAP`, 0.1, the weakest error of every +algorithm's list, so it never excludes a listed θ. For relative errors `x` +and `y` with `y ≤ b`, `(1 + x)(1 + y) − 1 ≤ (1 + b)·x + y`, and a ratio of +relative errors `a` over `b` is within `(a + b)/(1 − b)`. A relative bound +below 1 keeps zero at zero and keeps the sign, so a zero denominator gives the +same non-finite answer exactly and approximately. The ratio rule relies on +each operand's own relative guarantee (for DDSketch, its estimator contract: +one-signed values within the mapping's range). A sum or difference of +relative errors has no relative bound (cancellation), and a rank error (KLL) +has no value bound, so neither passes through division. Likewise, an exact +sum above approximate values retains their uncertainty: exact arithmetic does +not recover information lost below it. + +Probability is not propagated through operators. A query's failure +probability is split into a per-value one for each readout; a composed query +whose readouts' failure events add up (a ratio of two quantiles) reads each at +a level of a failure-probability grid, and the levels sum within the query's +per-value failure probability by the union bound, without assuming +independence between child errors. Estimator-local assumptions, such as +independent bucket hashing, must be stated separately. Shared DAG nodes do +not create independent errors merely because several consumers reference +them, and sharing does not justify reducing the failure budget. Per-result +guarantees also do not imply a simultaneous guarantee across every query or +evaluation time in a dashboard. + +## Readout requirements for composed queries + +Pass 2 gives each readout the accuracy it is required to meet. A query's +target error is divided by the factors `β` of the consumers on the path from +the readout to the query root, and capped by the validity caps on the way: the +largest error the readout's output may have for that query. + +- A query whose path reads one readout gives one requirement level: the + target divided by the path's factor, at the target's failure probability per + value: a whole-result δ split over the larger of `n_q` and the readout's + own values per evaluation (`Memo::max_values_per_evaluation`: at most the + series scanned below it, the declared distinct values of its state's keys, + one without keys), since a reduction such as `max` fails when any readout + it reads does. `SELECT count(DISTINCT dst) FROM flows` at δ = 0.01 reads + one distinct count, so its readout keeps 0.01. +- A composed query (several readouts, such as a ratio) gives every level of + the error grid (10%, 5%, 2%, 1%, 0.5%, 0.2%, 0.1%) and the failure-probability + grid (powers of ten from 1e-2 down to 1e-5, continued down to the first + one at or below a tenth of the query's per-value failure probability) not + looser than the cap. The logical-edge check then + keeps the levels a producer's θ meets. +- A readout reached by several queries gets the union of their levels. +- A count reads no value (`β = 0`): the readouts below it get no level + through it. +- A query whose target sets a failure probability with no per-value one + (`n_q` or the readout's values per evaluation unknown) gives no level. A + readout no query gives a level is pruned + (`RootPruneReason::ValuesPerEvaluationUnknown`); one another query gives a + level stays, and the query reads its group only exactly. + +The metric rule cuts a readout's edge when no query reaching it accepts its +metric, a union over the queries. Per query, Pass 2 also lists the groups it +may read only exactly +(`LogicalASAPMemoCandidates::exact_only_groups_of_query`, from +`ErrorMetricsOfMemo::exact_only_groups_of_query`), walking the live +candidates: the groups it reaches through a consumer without a contract for +its target's metric, and the readout groups whose error metric its target +does not allow. With Q0 = `p99 / p50` and Q1 = `p99 / p50 > 1` sharing the +ratio, Q0 may read the DDSketch readouts while Q1 reads the ratio and every +group below it only exactly; with `quantile_over_time(0.9, m[5m])` asked once +at a rank target and once at a relative one, each reads the other's readout +groups only exactly. Stage 3 must enforce the list per query; Pass 2 only +computes it. A scalar root's operands are roots: `scalar(v)` and a scalar +subquery keep the target's metrics, and any other scalar expression reads its +operands only exactly. + +The list is per group, so it is conservative where one group holds +candidates a query may and may not read: `avg by (job)(count_over_time(m[1m]))` +at an absolute target reads `avg` with a contract and the exact accumulator's +sum and count without one, so the group below is exact-only for that query +even through `avg`. A finer list needs Stage 3 to enforce the metric rule per +query on each edge. + +A PromQL subquery (`[10m:30s]`) has no error contract, so every sketch +readout below one serves no query with a target (sound). A contract for it +must split the failure probability over the subquery's inner steps (series × +range / resolution), not over the series alone. + +An exact target gets no approximate level. A model with only a fixed 1% +failure contract cannot meet a level that needs a smaller failure probability; +another supported configuration or candidate is needed. ## Evidence and trust boundaries @@ -285,30 +342,16 @@ certification. If a model uses an empirical accuracy calibration, the contract must identify its confidence level and scope rather than silently promoting an observation into a guarantee. -### Example: bounded Classic HLL - -A deployment declares `DataWorkload.sources[source].max_distinct_per_evaluation` -for the source the distinct count reads. It asserts an enforced maximum -distinct population per evaluation, including all merged tumbling windows; the classic -estimator and independent uniform bucket hashing are the model's -assumptions. Stage 1 sizes the precision from the query or allocated local -target and leaves the failure probability symbolic, as -`classic_hll_failure_probability[p=P,eps=ε](max_distinct[source])`. Pass 1 -then raises the precision when the source's declared bound needs it (for its -share of the failure probability), and Stage 3 binds the failure probability -from that bound in the normal selection checks. A bound outside the -contract's range gives a failure probability of 1; a missing bound leaves it -unknown, so the target is not met. - -The current model supports maxima from 1 to 131,072 (2^17) and precisions from 4 to 18, -and certifies only configurations that remain in the linear-counting branch. -It bounds collisions across every integer cardinality in the declared domain -and bounds overestimation deterministically. It is not an RSE-to-normal -conversion, nor does it cover HIP/MLE or arbitrary unbounded populations. - -Missing evidence leaves generic HLL confidence unknown. Invalid or infeasible -contracts cannot authorize the result. The contract does not certify another -estimator, another expression or an unsupported shared-grid grouping. +### Example: HLL distinct counts + +HLL's guarantee is its relative standard error, `1.04 / √m` for `m` buckets, +with an unknown failure probability. Its readout's metric is relative error, +but no failure probability can be evaluated, so a distinct count read from +HLL serves no query with an accuracy target +(`RootPruneReason::NoGuarantee`); KMV's distinct count, whose bound does not +depend on the number of distinct items, serves such a query. Missing evidence +leaves generic HLL confidence unknown. Invalid or infeasible contracts cannot +authorize the result. ## Sharing across consumers @@ -343,12 +386,12 @@ These percentages bound rank displacement, not latency-value error. ```mermaid flowchart LR - Input[Same source, filter and window] --> Shared[KLL computation per service sized for A] + Input[Same source, filter and window] --> Shared[KLL computation per service meeting A] Shared --> A[Consumer A: 1% rank error] Shared --> B[Consumer B: 5% rank error] ``` -B's independently sized candidate remains available. Global selection compares +B's own candidate, a θ meeting only B, remains available. Global selection compares that candidate with the reuse candidate and preserves the dependency on A's tighter computation if it chooses reuse. Sharing is therefore an explicit planning choice, not an unconditional rewrite or two separately charged builds. @@ -359,18 +402,10 @@ not just accuracy. Nor does this example justify serving B at a failure target of 0.1%; A's 1% failure guarantee would not meet it. A separate supported reuse rule or a different estimator configuration would be needed in those cases. -When the consumers' failure targets differ, the summary-capability rule offers -such a configuration. A shared summary states one accuracy bound at one failure -probability, so it is sized at the smallest one. Each consumer's parameters are -resized to it at no larger error, and the largest is kept. Suppose A asks -for 1% rank error at a 1% failure probability (`k = 269` alone) and B for -5% at 0.01% (`k = 79` alone). At 0.01% the Gaussian scale is 1.51, so A's -`k = 269` would give 1.5% rank error there, which misses A's target. A's -`k` is therefore grown to 412 (rank error 0.997% at 0.01%), and the shared KLL is -`k = 412` at 0.01%, which meets both. A distinct-count UnivMon's heap is shared -the same way. Checking each consumer at its own failure probability against -a smaller shared summary would need a guarantee per consumer, which the -current composition does not carry. +When the consumers' failure targets differ, each reads the shared KLL at +its own required failure probability: a θ whose guarantee meets both +requirements serves both, and one meeting only the looser does not serve +the tighter consumer. No θ is resized for sharing. ## Organization and extension contract @@ -380,36 +415,44 @@ The `asap_accuracy` crate separates these responsibilities: accuracy/src/ ├── lib.rs # Re-exports: the public interfaces ├── evidence.rs # AccuracyEvidence: the deployment's measured accuracy -├── requirements.rs # producer_satisfies_logical_requirements (α^log), target_limit -├── error_propagation.rs # ErrorPropagation: a consumer's output error from its inputs' -├── propagation_stats.rs # Evidence and source contracts -├── composition.rs # Error propagation across computations -├── budget.rs # Splitting a query's accuracy budget across approximate inputs -├── function_rules.rs # Error composition of scalar functions over approximate inputs +├── error_contract/ # ErrorContract: each operator's error contract per +│ # metric +├── metric_rule/ # ErrorMetricsOfMemo: the contracts, allowed metrics and +│ # readout metrics of one memo state; the metric rule per +│ # query (exact_only_groups_of_query) +├── readout/ # readout_error_metric: the metric of a readout's error +├── readout_caps/ # readout_levels → ReadoutRequiredAccuracy: caps through +│ # the error contracts, each readout's required-accuracy +│ # levels; MAX_ERROR_LEVELS +├── logical_edge/ # ErrorMetricsOfMemo::edge_meets_logical_requirements +│ # (α^log); within_limit, the requirement comparison └── guarantee/ - ├── mod.rs # Family/evaluation dispatch and source-contract integration - ├── kll.rs + ├── mod.rs # theoretical_guarantee (one lookup at a failure + │ # probability per algorithm) + ├── binomial.rs # binomial_upper_tail + ├── kll/ ├── ddsketch.rs - ├── hll.rs # Generic HLL and bounded Classic HLL - ├── cms.rs - ├── count_sketch.rs - ├── cardinality.rs # KMV model - └── univmon.rs + ├── hll.rs # HLL (unknown failure probability) + ├── cms/ + ├── count_sketch/ + ├── cardinality/ # KMV model + └── univmon/ ``` -Each estimator module owns its local accuracy formulas and associated sizing -formulas. The unified entry point delegates to these models; it does not +A directory is a module (`mod.rs`) with its unit tests beside it +(`tests.rs`). Each estimator module owns its local accuracy formulas: the +guarantee of a listed θ at a failure probability. The unified entry point delegates to these models; it does not contain a second copy of their mathematics. Serializable guarantee and metric types live in `asap-types` so planning, explanations and downstream binding share the same contract. -Adding an estimator or composition requires: +Adding an estimator or an operator's error contract requires: 1. A precisely defined error metric, estimator/evaluation semantics and assumptions. -2. Sizing behavior and a guarantee derived from the committed parameters, including unsupported parameter domains. +2. A θ list and a guarantee derived from each listed θ at a required failure probability, including unsupported parameter domains. 3. Explicit evidence requirements, population scope and provenance. 4. Propagation rules where supported; rejection or retained unknowns elsewhere. -5. Tests from target and evidence through sizing, composition and selection, including missing evidence and infeasible targets. +5. Tests from target and evidence through the guarantee at the required failure probability, error propagation and selection, including missing evidence and infeasible targets. Algorithm-specific mathematics belongs behind this common contract. It does not require a new selection rule for every sketch. A deployment's measured diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 6b7fe81d9..0435485ef 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -112,13 +112,22 @@ Operator nodes carry no physical properties: the logical DAG has no timing or lifetime. Materialization ([#509 §Materialization](../proposals/planner-layering.md#materialization)) decides, for each sub-DAG, whether its output is materialized, and if so with which -[`PhysicalProperties`](../../../crates/types/src/stages/physical/properties.rs): a +[`PhysicalProperties`](../../../crates/types/src/stages/physical/properties/mod.rs): a timing and a `Lifetime` (#1200). The lifetime is `None` (discarded after its reader), `OneEvaluation(medium)` (materialized until every reader of one evaluation finishes) or `AcrossEvaluations(medium)` (materialized for later evaluations), with `StorageMedium::Memory` or `StorageMedium::Disk`; an ingestion-time output cannot be materialized -for `OneEvaluation`. A -[`PhysicalPropertiesAssignment`](../../../crates/types/src/ir/properties/timing.rs) records, +for `OneEvaluation`. + +> **Note.** The code no longer has `PhysicalPropertiesAssignment`, +> `apply_physical_properties` or the test helpers named below: each Stage 2 +> [`PhysicalCandidate`](../../../crates/types/src/stages/physical/candidate.rs) +> carries its own `PhysicalProperties`, and +> `producer_satisfies_physical_requirements` (same file) checks timing and +> lifetime on each physical edge. The rest of this section describes the +> earlier assignment design. + +A `PhysicalPropertiesAssignment` records, per sub-DAG, the properties of its materialized output, named by the node whose output would be materialized. Any node may be named: a summary state, a population, or, as in #509 Example 4, the tumbling windows a query reads. `set(node, properties)` diff --git a/docs/design_docs/decisions/concat-unique-keys.md b/docs/design_docs/decisions/concat-unique-keys.md index 995eb99a6..1883b0fac 100644 --- a/docs/design_docs/decisions/concat-unique-keys.md +++ b/docs/design_docs/decisions/concat-unique-keys.md @@ -75,7 +75,7 @@ Both current `Concat`-constructing call sites, and every consumer of test fixtures) — none of them builds a fresh `Concat` with a `Dedup` on top that this feature could remove. - **Every consumer of `Schema::unique_keys`** in the DAG, to check for a - cost beyond "a literal `Dedup` node": `share_common_sub_dags` (now `Memo::find_or_add_group`, `crates/types/src/stages/logical/memo/hash_consing.rs`) + cost beyond "a literal `Dedup` node": `share_common_sub_dags` (now `Memo::find_or_add_group`, `crates/types/src/stages/logical/memo/hash_consing/mod.rs`) (gates CSE producer-sharing on `Schema::has_unique_key()`) and `asap_aware_mapping::rollup::is_legal_rollup_source` (gates rollup-source legality the same way, on an *`Aggregate`'s* own output schema). Neither diff --git a/docs/design_docs/proposals/asap-aware-mapping/end-to-end-accuracy-guarantees.md b/docs/design_docs/proposals/asap-aware-mapping/end-to-end-accuracy-guarantees.md index a272f8905..7d6dd0ca9 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/end-to-end-accuracy-guarantees.md +++ b/docs/design_docs/proposals/asap-aware-mapping/end-to-end-accuracy-guarantees.md @@ -1,7 +1,19 @@ # Design: End-to-End Accuracy Guarantees -> Status: partially implemented design. Typed guarantees, supported composition -> rules and accuracy gating are implemented. Empirical-input and combined +> Status: partially implemented design. Implemented today: typed accuracy +> targets and per-family guarantees (`AccuracyTarget`, `AccuracyGuarantee`); +> per-operator error contracts (`ErrorContract { error_factor_per_input, +> linear_bound_holds_up_to_per_input }` in +> [`crates/accuracy/src/error_contract/mod.rs`](../../../../crates/accuracy/src/error_contract/mod.rs)); +> readout requirements +> ([`crates/accuracy/src/readout_caps/mod.rs`](../../../../crates/accuracy/src/readout_caps/mod.rs)); +> and the logical edge check +> ([`crates/accuracy/src/logical_edge/mod.rs`](../../../../crates/accuracy/src/logical_edge/mod.rs)). +> The guarantee composition and propagation pipeline this proposal describes +> (`CompositionOperator`, `AccuracyModel`, `AccuracyBudgetAllocator`) is not +> in the code: the per-operator error contracts replaced it. The +> [accuracy models concept](../../concepts/accuracy-models.md) describes the +> current model. Empirical-input and combined > parameter configuration remain extensions as described below. The > [implementation companion](../../../develop_docs/end-to-end-accuracy-guarantees.md) > describes current contracts and validation. @@ -209,6 +221,11 @@ AccuracyModel::satisfies accepts or rejects the candidate before CostModel ranking ``` +> **Replaced.** This pipeline and the components table below are not in the +> code: per-operator error contracts (`ErrorContract`), readout requirements +> and the logical edge check replaced guarantee composition; see the +> [accuracy models concept](../../concepts/accuracy-models.md). + Four components have distinct responsibilities: | Component | Responsibility | Does not do | @@ -244,6 +261,11 @@ struct AccuracyGuarantee { Summary state does not itself claim a final-result guarantee. The guarantee is attached to a finalized value, such as a `SummaryEstimate`. +> Implemented: `AccuracyGuarantee` holds only an `AccuracySpec` with a +> constant bound (`BoundExpr::Constant`) and a failure probability +> (`ProbabilityExpr::{Zero, Constant, Unknown}`); the symbolic combinators +> and the provenance trail below are not in the code. + #### Error metrics The built-in model distinguishes: @@ -296,6 +318,10 @@ and checks the final query result against its target. The budget allocator only proposes allocations; it does not prove them. `CostModel` may rank only candidates that survive the complete accuracy check. +> **Replaced.** The composition rules below are now per-operator error +> contracts (`ErrorContract`); an operator without a contract for a metric +> fails the edge (the metric rule). + Composition rules are explicit and metric-aware. The built-in model supports only registered exact, additive, Lipschitz, relative, and exact-aggregation rules. It uses union bounds rather than assuming independence. Cross-metric and @@ -410,6 +436,10 @@ epsilon_i = epsilon_total / approximate_layer_count delta_i = delta_total / approximate_layer_count ``` +> **Replaced.** No budget allocator exists in the code: readout requirements +> (`readout_caps.rs`) pass each query's target down through the consumers' +> error contracts to the required accuracy of each readout. + Every allocated candidate is reconfigured, propagated, and checked. Allocation is only a proposal, and recording a requested delta in metadata is not evidence that an algorithm achieves it. diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 84d7dd22e..eac375728 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -117,7 +117,7 @@ A node in the physical data will represent the data or summary instance, so a no | How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | | How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `GroupProperties::aggregate_coverage`, derived in `stages/logical/aggregate_coverage.rs` | §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, derived in `stages/logical/group/aggregate_coverage.rs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -237,7 +237,7 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -Coverage is a memo-group property, derived like an output schema and never declared; its types and derivation are in `crates/types/src/stages/logical/aggregate_coverage.rs`. The memo derives it once, when `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`): `aggregate_coverage` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s, and the result, its inputs named by `MemoGroupId`, is stored in the group's `GroupProperties::aggregate_coverage`. Node trees have none. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). +Coverage is a memo-group property, derived like an output schema and never declared; its types and derivation are in `crates/types/src/stages/logical/group/aggregate_coverage.rs`. The memo derives it once, when `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`): `aggregate_coverage` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s, and the result, its inputs named by `MemoGroupId`, is stored in the group's `GroupProperties::aggregate_coverage`. Node trees have none. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). ```rust /// What a summary state computes, with every lifted selection removed, @@ -951,7 +951,6 @@ OperatorNode §6.1 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output │ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) │ (θ, `SketchParams`, is on the candidate) §6.4 -└── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 └── definition: AggregateDefinition { input, unlifted_operators } @@ -966,7 +965,7 @@ MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | | §6.6 Aggregate coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of aggregate memo groups | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. The memo derives the `SummaryAgg` group's `AggregateCoverage` (§6.6) and stores it in the group's `GroupProperties::aggregate_coverage`. +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5). The memo derives the `SummaryAgg` group's `AggregateCoverage` (§6.6) and stores it in the group's `GroupProperties::aggregate_coverage`. ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) @@ -982,9 +981,6 @@ pub struct OperatorNode { pub result_kind: OperatorResultKind, /// The output columns. Derived from `operator` and its children. pub schema: Schema, - /// The accuracy statement of the output, once known. `None` does not - /// mean exact. - pub guarantee: Option, } ``` @@ -1353,9 +1349,9 @@ pub struct CurrentSeriesInput { } ``` -A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. +A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It describes evaluation and finalized values, and `MaintainPopulation` values (whose population is exact); never sketch or accumulator state. No node type stores it today. -### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; `SourceColumn` in `crates/types/src/ir/operator/arguments.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/column_properties.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/stages/logical/group/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; `SourceColumn` in `crates/types/src/ir/operator/arguments.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/group/column_properties.rs`; the plan's content id in `crates/types/src/plan/view_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. diff --git a/docs/design_docs/proposals/joint-planning-implementation-plan.md b/docs/design_docs/proposals/joint-planning-implementation-plan.md index 39119c768..e00c7533b 100644 --- a/docs/design_docs/proposals/joint-planning-implementation-plan.md +++ b/docs/design_docs/proposals/joint-planning-implementation-plan.md @@ -13,7 +13,7 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc | Scope | Current implementation | Responsibility | | --- | --- | --- | | Logical identity | `stage0-memo/src/lib.rs`; `types/src/stages/` | `Memo`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`, hash-consed by `Memo::find_or_add_group`), stable initial query/group references. | -| Computation and sharing rules | `stage1-pass2-reuse-candidates/src/{lib.rs,rules.rs,rules/}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | +| Computation and sharing rules | `stage1-pass2-reuse-candidates/src/{lib.rs,rules/}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | | Physical legality | `stage2-physical-optimization` | Executable operators, timing and materialization eligibility. | | Candidate dependency inventory | `stage2-physical-optimization/src/lib.rs`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning; Stage 3 builds the selected plan (`PhysicalASAPDAG`) directly. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`stage3-plan-selection/src/search/candidates/scopes.rs`). | | Solver | `stage3-plan-selection/src/search/milp` | Activation, producer/root assignment, reachability, acyclicity, resource constraints, and solver warm starts. | diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index eedea2fb3..1a7b85570 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -283,7 +283,7 @@ one. It makes no implementation choice, so this section counts it as Stage 0 lists `memo_of_queries` with Pass 1. `memo_of_queries` (`crates/stage0-memo/src/lib.rs`) visits the frontends' roots children first and finds or adds each node's group with `Memo::find_or_add_group`, whose -expression equality is in `crates/types/src/stages/logical/memo/hash_consing.rs`. Two structurally equal nodes +expression equality is in `crates/types/src/stages/logical/memo/hash_consing/mod.rs`. Two structurally equal nodes become one group only when sharing them is legal, that is, when every reader of the shared node reads the rows it would read alone. Equal nodes are interned children first, so they always read one input, and a node is legal @@ -311,19 +311,9 @@ rule's origin, and gives each reader a candidate that reads that group. | `OriginOfCandidateForMemoGroup` | Added by | Candidates | |---|---|---| | `Pass1` | `pass1_memo` (`stage1-pass1-per-computation-candidates`: `computation_targets.rs`, `candidates/`) | Every Pass 1 way to compute the group's measures: exact, exact accumulator, each summary algorithm with one build per configuration of its predefined list (`parameter_list`), and fused computations. A fused candidate lists the groups of the operators it replaces in `fused_groups`; those groups keep their own candidates for their other readers. | -| `TumblingWindows` | Window composition: `add_local_window_composition` and `extend_local_window_composition` (`pass2/memo/window_candidates.rs`) | A window summary computed from tumbling windows, at each tumbling-window width the reader's evaluation grid allows. The tumbling-window chain stays private to the candidate. | -| `WidenedScans` | Scan widening: `add_scan_label_candidates` (`pass2/memo/rules/scan_labels.rs`) | A reader of a metric's scans widened to a shared label set, read through a `Project`. | -| `SummaryInput` | `factor_summary_inputs` (`pass2/memo/rules/summary_inputs.rs`) | Canonical value preprocessing feeding a summary build. | -| `TopKReuse` | Exact top-k reuse: `add_topk_candidates` (`pass2/memo/rules/topk.rs`) | One ranking (`Limit(k_max)` over `Sort`), and each top-k reading its first `k` rows from it. | -| `Rollup` | Rollup: `add_finer_aggregate_candidates` (`pass2/memo/rules/finer_aggregate.rs`), `add_rollup_candidates` and `add_window_rollup_candidates` (`pass2/memo/rules/rollup.rs`) | A coarse group computed from a finer group's values or states. | -| `SummaryResult` | `factor_summary_results` (`pass2/memo/rules/summary_results.rs`) | A summary readout factored out of a composite implementation, so that other readers can read it. | -| `SharedBuild` | Summary capability: `add_shared_summary_state_groups` (`pass2/memo/state_groups.rs`), Pass 2's last step | For each build made two or more times (by different candidates, or twice within one), one concrete summary-state group with one `SharedBuild` candidate; and for each candidate making such builds, one `SharedBuild` copy that reads the groups instead of building its own states. | - -The state-configuration rule (`pass2/memo/rules/shared_configuration_readers.rs`) runs just -before the shared builds are found. It rewrites readers to each shared size -and to the newest tumbling windows of each tumbling-window chain layout they may read, so that -equal builds appear, and a rewrite keeps the origin of the candidate it -rewrites. +| `TumblingWindows` | Chain groups: `add_chain_groups` (`stage1-pass2-reuse-candidates/src/rules/chains.rs`) | A chain group `SummaryTumblingWindowChain[w](X)` per width the reaching queries' evaluation grids allow, one build per surviving θ, the `TimeShift`/`TimeRange` groups over it, and the `SummaryMerge` over them in the state group. A chain's edge into a group whose evaluation grid its width does not divide fails with `ChainWidthDoesNotDivideEvaluationGrid`, so the merge over it is pruned: a 15 s chain never serves a state also read every 10 s. | +| `TopKReuse` | Exact top-k reuse: `add_topk_candidates` (`stage1-pass2-reuse-candidates/src/rules/topk.rs`) | For each exact top-k, `Aggregate[TopK k]` over the ranked rows of the top-k group of the same input and partition with the largest `k`, partitioned by that group's leading key columns: the `k` highest of the `k_max` highest rows are the `k` highest. | +| `Rollup` | Rollup from the finest common group: `add_rollup_candidates` (`stage1-pass2-reuse-candidates/src/rules/rollup/`) | A coarse group computed from a finer group's values or states (through a compensating `Filter` on the finer group's keys), and the builds of a cluster's finest common group. | **Each stage over the memo.** Each stage keeps its decision, on groups instead of whole candidates: @@ -392,48 +382,29 @@ stands for 282 = 2 × (1 + 2 × 7 + 2 × 7 + 7 + 7 + 2 × 7 × 7) candidate DAGs: G6 exact or from accumulators, times G2 and G4 both exact, one exact and the other from KLL or DDSketch, both from one family, or one from each. -Steps 3 and 4, and their rows in the table after step 4, predate this Pass 1 -sizing; their counts are not recomputed here. - -**Step 3, Stage 1 Pass 2: window composition.** The groups stay the same 7. -G2 and G4 get 120 `TumblingWindows` candidates each (6 summary candidates × 20 tumbling window -widths, from 1 s to 60 s, that divide one minute), 127 candidates each in -all; the memo has 259 candidates. - -**Step 4, the other Pass 2 rules and the concrete summary-state groups** -(rules 2–9 of `generate_logical_asap_memo_candidates`, then -`add_shared_summary_state_groups`): -- 354 groups: the 7 above; 336 `SummaryTumblingWindowChain` groups (7 summary configurations - × 48 tumbling-window layouts, with tumbling-window widths from 20 ms to 60 s); 7 `SummaryAgg` - state groups (the 6 summaries above and a KLL with `k` 1156, - `failure_probability` 1e-7); 2 `SummaryAgg` groups for G6's sum and count - accumulators; and 2 `FinalizeExactAccumulator` groups (G6's sum and count - readouts). -- 1738 candidates. G2 and G4 have 691 each: 10 `Pass1` (the 7 above and - 3 copies of the KLL `k` 567 that the state-configuration - rule rewrote to the shared configurations with `k` 567, 1156 and 1184; the - `k` 1184 one composes the same DAG as the KLL `k` 1184), - 336 `TumblingWindows` and 345 `SharedBuild`. G6 has 3: `Pass1`, `SummaryResult` and - `SharedBuild`. -- Enumeration stops at the former `stage_pipeline`'s default limit of 1,000 candidate - DAGs (`--memo-enumeration-limit`), as it does after step 3. +**Step 3, Stage 1 Pass 2** (`generate_logical_asap_memo_candidates`). +The workload has no second aggregate of the same definition (no rollup), no +top-k (no top-k reuse) and no evaluation grid (no chain group), so the +groups and candidates stay 19 and 36. The query's target is an absolute +error, which the `+` carries to G4 and the division to neither of its +operands: the metric rule cuts every edge from a KLL readout (rank error) +or a DDSketch readout (relative error) into G2 and G4, so no readout gets a +required accuracy. Dead-candidate elimination removes those 4 candidates of +G2 and G4, the 4 readouts and the 14 builds (22 pruned); 14 candidates +survive, with 16 logical edges. The surviving choices are G6 exact or from +the accumulators, over exact quantiles: 2 candidate DAGs. | After | Groups | Candidates for memo groups | Candidate DAGs | |---|---|---|---| | Stage 0 | 7 | – (each group is its own node) | 1 | | Stage 1, Pass 1 | 19 | 36 | 282 | -| Stage 1, Pass 2 window composition | 7 | 259 | over 1,000 (enumeration stopped at its limit) | -| Stage 1, Pass 2 rules and concrete summary-state groups | 354 | 1738 | over 1,000 (enumeration stopped at its limit) | - -These numbers are the production planner's, read from the stage-pipeline -document `tools/dag-viewer/examples/memo-ratio-of-quantiles.json` (made by -the former `stage_pipeline` from `tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json`, -with accuracy guarantees enforced and the `aws-fargate-us-east-1` cost model). The Stage 1, Pass 1 row is -`memo_of_queries` then `pass1_memo` run on that workload with the -executor's summary families, at the current Pass 1 sizing. No test asserts -these numbers. The last -step is Pass 2's memo over the complete tumbling-window-width domain, which -gives 438 groups and 2158 candidates. +| Stage 1, Pass 2 | 19 | 36, 14 surviving dead-candidate elimination | 2 | + +These numbers are the production planner's: `memo_of_queries`, then +`pass1_memo` and `generate_logical_asap_memo_candidates`, run on +`tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json` with the +executor's summary families and accuracy guarantees enforced. No test +asserts them. #### Algorithm: memo to candidate DAGs @@ -477,7 +448,7 @@ that vector. | Reason | Effect on the count | In the example | |---|---|---| | Shared group | A group several chosen candidates read takes one candidate per choice, not one per reader. | G1 is read by G2 and G4 and chosen once. | -| Group needed only under some choices | A group joins the choice only when a chosen candidate reads it. | A `SummaryTumblingWindowChain` group is needed only when a `SharedBuild` candidate reading it is chosen; a choice of `Pass1` candidates needs none of the 347 added groups. | +| Group needed only under some choices | A group joins the choice only when a chosen candidate reads it. | The KLL state group G7 is needed only when a candidate reading G8 or G11 is chosen; a choice of exact candidates needs none of G7–G12. | | Fused candidate | The groups a fused candidate implements are computed inside it and never become pending. | None (the example has no fused computation). | In Pass 1 of the example the count is not a product either: G2 and G4 @@ -518,19 +489,21 @@ full tumbling-window-width domain, with no time limit. #### Memo code interface -The memo rules are in `crates/stage1-pass2-reuse-candidates/src/` (module -`memo`, crate-private: the crate root re-exports each entry point); the memo types Stage 1 hands Stage 2 are in +The memo rules are in `crates/stage1-pass2-reuse-candidates/src/` (files +`rules/`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs` and +`evaluation_grids.rs`, crate-private; the groups each query may read only +exactly come from `asap_accuracy::ErrorMetricsOfMemo`): the crate +root's `generate_logical_asap_memo_candidates` runs them); the memo types Stage 1 hands Stage 2 are in `crates/types/src/stages/` (`asap_types::stages`); the facade that runs the stages is in `crates/planner/src/run.rs`; the search over Stage 2's candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `groups/`, `milp/`). | Step | Entry point | Path | |---|---|---| -| Stage 0 | `memo_of_queries` → `Memo` (without candidates), `MemoGroup`, `MemoGroupId` | `stage0-memo/src/lib.rs` (the types: `asap_types::stages`); hash-consing is `Memo::find_or_add_group` in `types/src/stages/logical/memo/mod.rs`, its expression equality in `memo/hash_consing.rs` | +| Stage 0 | `memo_of_queries` → `Memo` (without candidates), `MemoGroup`, `MemoGroupId` | `stage0-memo/src/lib.rs` (the types: `asap_types::stages`); hash-consing is `Memo::find_or_add_group` in `types/src/stages/logical/memo/mod.rs`, its expression equality in `memo/hash_consing/mod.rs` | | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | -| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `stage1-pass2-reuse-candidates/src/lib.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | -| Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | +| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{candidates.rs,rollup/,topk.rs,chains.rs}`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`; the readout levels: `asap_accuracy::readout_levels`; the groups each query may read only exactly: `asap_accuracy::ErrorMetricsOfMemo::exact_only_groups_of_query`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | | Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -599,8 +572,7 @@ the memo path's entry points. |---|---|---|---| | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (no longer in the code) | Logical roots → Stage 0's `Memo` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support → the same `Memo` with each group's `CandidateForMemoGroup`s (a summary with one build per configuration of its `parameter_list`) | `memo_of_queries`, `pass1_memo` | -| 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | -| 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | +| 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → `LogicalASAPMemoCandidates`: the same memo with the rules' groups and candidates added (rollup, top-k reuse, chain groups), each readout's required accuracy, the logical edges between the surviving candidates, the pruned candidates and, per query, the groups it may read only exactly | `generate_logical_asap_memo_candidates` (`asap_stage1_pass2_reuse_candidates`) | | 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_stage3_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -1226,8 +1198,14 @@ agree, and the interval and a declared phase are multiples of the window: `m[5m]` every 5 minutes is one tumbling 5-minute window maintained at ingestion time and read at each evaluation, as a one-tumbling-window chain would be, while `m[20s]` every 20 s at phase 5 s, or every 30 s, is not kept whole (Pass 2 offers 5 s or 10 s tumbling windows). A -reader whose phase is not declared is taken to evaluate at multiples of its -interval, as for a tumbling-window chain. +reader whose phase is not declared has no known evaluation grid +(`QueryRecurrence::evaluation_grid_ms` is `None`: Prometheus staggers rule groups +by a hash, so a fixed interval does not mean phase 0), and Pass 2 adds no +tumbling-window chain for a state it reads. Below a PromQL subquery the +grid is the subquery's: `[10m:30s]` evaluates `` at the +multiples of 30 s whatever the outer grid, and `[10m:]` at the +server's default interval, which the planner does not know, so a state +there gets no chain. #### Physical operator implementation diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 340e25183..f3bf869c6 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,12 +17,12 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (execution timing, the selection algebra of aggregate coverage) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages`, with the memo groups' aggregate coverage and Stage 2's `PhysicalProperties` (timing and `Lifetime`)); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | +| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (`execution_data_state`: execution timing and what an edge carries; `selection`: the selection algebra aggregate coverage is stated in) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages::logical`: the memo, its groups with their aggregate coverage, candidates and logical edges; `stages::physical`: Stage 2's physical candidates with their `PhysicalProperties` (timing and `Lifetime`) and edges; `stages::pruning`); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-stage0-memo` (`crates/stage0-memo`) | Stage 0 hash-consing: the memo groups of the workload's query roots (`memo_of_queries`) | | `asap-stage1-pass1-per-computation-candidates` (`crates/stage1-pass1-per-computation-candidates`) | Stage 1, Pass 1: the initial candidates of each memo group | | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates`) | Stage 1, Pass 2: the reuse rules over memo groups and the logical edges | -| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, composition, budget split, target check), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`producer_satisfies_logical_requirements`); depends only on `asap-types` | +| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, each operator's error contract, the target comparison `within_limit`), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`ErrorMetricsOfMemo::edge_meets_logical_requirements`); depends only on `asap-types` | | `asap-stage2-physical-optimization` (`crates/stage2-physical-optimization`) | Stage 2: materialization and physical operator implementation | | `asap-cost` (`crates/cost`) | The cost model the stages share: resource-usage estimation (the analytical per-operator model, measured operation costs, state byte sizes) and the cost models (`ResourceUsage`, `CostModel`, `CostInputs`); depends only on `asap-types` | | `asap-stage3-plan-selection` (`crates/stage3-plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | @@ -32,8 +32,8 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with ```text asap-types ◄── frontends (Stage 0) -asap-types ◄── asap-cost (shared cost model) ◄── asap-stage2-physical-optimization, asap-stage3-plan-selection -asap-types ◄── asap-accuracy (shared accuracy model) ◄── asap-stage1-pass1-per-computation-candidates, asap-stage2-physical-optimization, asap-stage3-plan-selection +asap-types ◄── asap-cost (shared cost model) ◄── asap-stage3-plan-selection +asap-types ◄── asap-accuracy (shared accuracy model) ◄── asap-stage1-pass2-reuse-candidates, asap-stage3-plan-selection asap-types ◄── asap-stage0-memo (Stage 0 hash-consing) ◄── asap-planner asap-types ◄── asap-stage1-pass1-per-computation-candidates (Stage 1, Pass 1) ◄── asap-planner asap-types ◄── asap-stage1-pass2-reuse-candidates (Stage 1, Pass 2) ◄── asap-planner @@ -51,7 +51,8 @@ QueryRoots (Stage 0) ▼ Stage 1, Pass 1 Memo (each group's initial candidates) │ asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates - │ the memo rules (memo/rules/) over the candidate memo Stage 1, Pass 2 + │ the reuse rules (crates/stage1-pass2-reuse-candidates/src/rules/) + │ over the candidate memo Stage 1, Pass 2 ▼ LogicalASAPMemoCandidates │ asap_stage2_physical_optimization::generate_physical_asap_memo_candidates Stage 2 @@ -72,14 +73,15 @@ same pipeline behind the `OptimizationPass` trait. | --- | --- | | Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`); the candidates that compute them | `asap-stage1-pass1-per-computation-candidates`, `computation_targets.rs`, `candidates/` | | Which summary families serve an intent (`summary_candidates`) | `asap-stage1-pass1-per-computation-candidates`, `summary_realization.rs` | -| Pass 2 steps: rollup from the finest common group (shared producers across group keys), top-k reuse, readout requirements, dead-candidate elimination, chain groups (tumbling windows) and the logical edges | `asap-stage1-pass2-reuse-candidates`, `lib.rs` (the steps in order) and `rules/` (`rollup.rs`, `topk.rs`, `chains.rs`) | -| Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | none today (deleted in #1207 PR 0) | +| Pass 2 steps: rollup from the finest common group (shared producers across group keys), top-k reuse, readout requirements, dead-candidate elimination, chain groups (tumbling windows) and the logical edges | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates/src/`): `lib.rs` (the steps in order), `rules/` (`candidates.rs`, `rollup/`, `topk.rs`, `chains.rs`), `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs` and `evaluation_grids.rs`; the groups each query may read only exactly come from `asap_accuracy::ErrorMetricsOfMemo` | +| Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | none today (the standalone enumeration was deleted) | | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | | Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-stage2-physical-optimization` | -| Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | -| Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-stage3-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | -| The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | -| The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `requirements.rs`, `error_propagation.rs`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | +| Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | +| Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-stage3-plan-selection`, `lib.rs`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`) and `measured.rs` | +| The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | +| The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | +| The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | | What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | | The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `models.rs` (`PlanningModels`) and `pass.rs` (the trait); the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 8bbd9da5c..2de5b59f2 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -64,13 +64,14 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- (`AccuracyEvidence::measured_guarantee`) when there is one, else its algorithm's error formula (`theoretical_guarantee`); Pass 2 compares it with each readout's `RequiredAccuracy` - (`producer_satisfies_logical_requirements`). It fails closed: no rule + (`ErrorMetricsOfMemo::edge_meets_logical_requirements`). It fails closed: no rule means no guarantee, and the edge is not admissible. - A target with no `accuracy_metric` accepts a guarantee in any metric that meets its ε, δ and scope; Stage 3 reports the metric it accepted (`AcceptedAccuracy`). -- An approximate input split across several inputs gets its share through - `accuracy::budget::per_input_target`. +- A query that composes several readouts gives each readout a level of an + error and failure-probability grid, within the cap its path's consumers + (`asap_accuracy::ErrorContract`) leave it. - Facts come only from the declared `DataWorkload`; an undeclared fact is unknown, never zero. diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 43aa9641b..5cc594dd9 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -19,9 +19,9 @@ aggregate intent and AccuracyTarget -> choose and commit concrete summary parameters -> derive the local guarantee from those committed parameters -> realize the child and obtain its finalized guarantee - -> propagate guarantees through the registered composition rule + -> propagate required errors through the operators' error contracts -> check the final guarantee against AccuracyTarget - -> keep legal candidate or record AccuracyError + -> keep legal candidate or record the pruning reason -> Stage 3 prices the valid candidates under the cost model (cost does not prove accuracy) ``` @@ -75,79 +75,101 @@ pub trait AccuracyEvidence { } ``` -`accuracy::budget::per_input_target` splits a target equally among the -approximate inputs of one composition (ε / n for an additive metric, -1 − (1 + ε)^(−1/n) for a relative one, δ / n). The relative share meets the -composed bound of one product or quotient of two inputs: a ratio of two -relative errors `a` and `b` is within `(a + b) / (1 − b)`, so ε = 0.05 gives -each side 0.0241. A nested quotient (`x / (y / z)`) or an exact numerator over a -lone approximate denominator composes a bound above ε and is rejected. Every candidate must then be resized, propagated, and -checked before being treated as satisfying the target. This is distinct from +Pass 2 gives each sketch readout the accuracy it is required to meet: a +query's target divided by the propagation factors of the consumers on the path +from the readout to the query root (`asap_accuracy::ErrorContract`, a linear +bound per operator and metric, holding on an operand only up to an error +where it is limited, such as the denominator of a ratio). A query that +reads one readout gives it one level; a query that composes several readouts +(a ratio of two quantiles) gives every level of an error and failure-probability +grid not looser than the cap, and each logical edge is then checked against +the levels. A consumer with no contract for a metric fails the edge from a +readout in that metric (`RootPruneReason::MetricRule`), so an +approximate value is never treated as exact. Every candidate is +checked against the target before being treated as satisfying it. This is distinct from candidate visibility: a Stage 1 candidate whose output has no guarantee is still enumerated and appears in the stage document, but Stage 3 rejects it for any approximate target. Presence and cost are not accuracy certification. See the [workflow design](../design_docs/architecture/input-output-workflow.md#planning-evidence-inputs) for this boundary. -## Composition contracts +## Error contracts -### Exact values - -An exact value contributes zero error and zero failure probability: - -```text -B_exact = 0 -delta_exact = 0 -``` - -An exact input does not exempt a local approximate sketch from target checking. - -### Additive and Lipschitz composition - -Compatible absolute bounds compose without an independence assumption: +Only a sketch readout (`SummaryEstimate` over a sketch state) introduces +error. Every other operator passes its inputs' errors on through its error +contract, `ErrorContract` (`accuracy/src/error_contract/mod.rs`, one per +candidate and metric in `ErrorMetricsOfMemo`): a linear bound on the output +error +from the input errors in one metric, ```text -B_total <= B_input + B_local -delta_total <= delta_input + delta_local +error_output <= sum_i factor_i * error_i while error_i <= validity_cap_i ``` -For a registered `L`-Lipschitz transformation: - -```text -B_output <= L * B_input + B_local -delta_output <= delta_input + delta_local -``` - -Failure probabilities use a union bound. - -### Relative composition - -For a registered multiplicative rule whose values are known to be -non-negative: - -```text -epsilon_total = - epsilon_input - + epsilon_local - + epsilon_input * epsilon_local -``` - -Reject the rule when sign information is missing or values may cross zero. - -### Exact aggregation over approximate values - -For an exact sum, sum the input bounds and union-bound input failures. If the -number of folded rows is required but unknown, keep the result symbolic. - -For exact minimum or maximum, use the maximum input value-error bound and -union-bound failures. This does not prove the identity of a winning key. - -### Unsupported composition - -Accept approximate-over-approximate composition only when a rule exists for -the operator and metric. Cross-metric and unregistered same-metric composition -must return `UnsupportedComposition`; never treat an approximate child as -exact. +or `None` when the operator has no contract for that metric. A contract +keeps the metric: an input's error and the output's are in the same one. + +| Operator | Metric | Factor per input | Validity cap | +| --- | --- | --- | --- | +| `Sort`, `TimeRange`, `TimeShift`, `Concat`, a readout, a finalize, `SummaryMerge` | any | 1 | none | +| `Filter` whose predicate reads only columns copied from a source (labels, keys) | any | 1 | none | +| `Project` whose items reading a computed value (not a copied label, the time or the series identity) are copies | any | 1 | none | +| the same, with items computing `abs`, `clamp`, `clamp_min`, `clamp_max` (or PromQL's `promql_abs`, …) or a negation of a column | absolute | 1 | none | +| the same, with items multiplying or dividing a column by a nonzero literal | relative | 1 | none | +| `Aggregate` (or exact accumulator build) by keys copied from a source, without measure filters or `HAVING`: `count` | any | 0 | none | +| the same: `min`, `max` | absolute, relative | 1 | none | +| the same: `avg` | absolute | 1 | none | +| `+`, `-` | absolute | 1 | none | +| `/` | relative | `1 / (1 - b)` on both | `b` on the denominator | +| `*` | relative | `1 + b` on the left, 1 on the right | `b` on the right operand | + +`b` is `RELATIVE_OPERAND_VALIDITY_CAP` (0.1, the weakest error of every +parameter list). Every other operator has no contract: a filter or grouping +on a computed value, a ranking, a `sum`, a comparison, any other function. + +The contracts are used in three places: + +- **The metric rule** (`asap_accuracy::ErrorMetricsOfMemo`, built once per + memo state before an edge pass, `accuracy/src/metric_rule/mod.rs`): a + query root accepts its target's metric + (every metric when the target names none); an input of a candidate + accepts the metrics its output accepts that the candidate's contract + carries. An edge from a readout whose metric its consumer does not + accept fails (`RootPruneReason::MetricRule`), so an approximate value + never reaches an operator with no contract for its error. Where no query + with a target reaches a group, no metric rule applies. The rule is a + union over the queries reaching a group; per query, Pass 2 lists the + groups reached through a live consumer without a contract for the query's + metric, and the readout groups whose error metric the query's target does + not allow (`ErrorMetricsOfMemo::exact_only_groups_of_query`), which Stage 3 + must let that query read only exactly. +- **Readout requirements** (`asap_accuracy::readout_levels`, + `accuracy/src/readout_caps/mod.rs`, a `ReadoutRequiredAccuracy` per + readout): each query's target is passed from its + root down to the readouts, divided by each consumer's factor and capped + by its validity cap, keeping per group the largest cap of each query + and metric. A query that reads one readout gives it one level, its cap + at the target's failure probability per value (a whole-result δ split + over the larger of the values one evaluation returns and the readout + group's values per evaluation, `Memo::max_values_per_evaluation`, since a + `max` over many readouts fails + when any of them does); a count passes no cap down; a query that combines several + readouts (a ratio of two quantiles) gives every level of the grid + `MAX_ERROR_LEVELS` × a failure-probability grid (powers of ten from 1e-2 + down to 1e-5, continued to the first at or below a tenth of the per-value + δ) that is not looser than the cap and the per-value δ. Pass 2 writes each level on a readout + candidate of its own. +- **$\alpha^{\log}$** (`ErrorMetricsOfMemo::edge_meets_logical_requirements`, + `accuracy/src/logical_edge/mod.rs`): an edge into a readout holds when the + producer's θ has a guarantee (measured, else `theoretical_guarantee`) + in the required metric within the required error at the required + failure probability, and a merged state keeps the guarantee of a build. + Under a query with a target, a readout with no guarantee, or one whose + failure probability is unknown (HLL's), fails with `NoGuarantee`. + +A value that is exact has no error and needs no contract. Nothing composes +failure probabilities across readouts beyond the per-value split and the +grid above, and nothing adds a local error at a non-readout operator. ## Built-in sketch contracts @@ -199,18 +221,11 @@ epsilon_cardinality = 1.04 / sqrt(2^p) delta = unknown ``` -Sizing inverts the magnitude. HLL's current parameter model contains precision -only; it does not encode a confidence-level budget. Generic HLL may therefore -satisfy a target with only `error` set, but not one that also sets -`failure_probability`. - -Classic HLL (its linear-counting branch) has an estimator-specific confidence -bound (`accuracy/guarantee/hll.rs`). Its precision is sized from (ε, δ), and -its failure probability stays symbolic, -`classic_hll_failure_probability[p=P,eps=ε](max_distinct[source])`, until -plan selection binds it with the source's -`DataWorkload.sources[source].max_distinct_per_evaluation`. A missing bound -leaves the target unmet. +HLL's parameter model contains precision only; it does not encode a +confidence-level budget. With its failure probability unknown, a distinct +count read from HLL serves no query with an accuracy target (the edge fails +with `NoGuarantee`); KMV is the distinct-count summary with a certified +bound. ### KMV and Theta @@ -261,27 +276,17 @@ with probability at most `1/3`. Zero or even depth has no modeled guarantee. ASAPPlanner contains the guarantee algebra and parameter-derived contracts. Planner is not a query executor. Data- or runtime-dependent evidence enters through `AccuracyEvidence` (measured guarantees) or the data workload's -declared facts, is exposed to propagation as typed `PropagationStats`, and -is recorded in provenance. The default evidence has no measurements. Guarantee derivation remains conservative. +declared facts, and is recorded in provenance. The default evidence has no measurements. Guarantee derivation remains conservative. ### TopK membership -`PropagationStats` supplies: - -- the lower confidence bound of the kth selected item; -- the greatest upper confidence bound among excluded items; and -- the union-bound failure probability of all certificate intervals. - -The intervals must already include underlying sketch error. Certify exact -membership only when: - -```text -selected_kth_lower_bound > excluded_max_upper_bound -``` - -The result uses `TopKMembership`, zero membership error, and the supplied -failure probability. Missing, non-finite, invalid, equal, or overlapping -evidence rejects the candidate. +Planner has no membership certificate for a top-k. A top-k readout from a +heap variant (`CmsWithHeap`, `CountSketchWithHeap`) is admitted when its +heap holds at least `k` entries (else the edge fails with +`RootPruneReason::HeapSmallerThanTopK`) and the counts it reports meet the +requirement: the Count-Min L1 or Count Sketch L2 absolute stream bound of +the sketch's width and depth. Whether the right `k` keys are selected is not +checked on its own. ### Hydra shared grid @@ -296,6 +301,8 @@ When observations are unavailable, preserve `hydra_shared_grid_collision_bound` and `hydra_shared_grid_failure_probability` as symbolic leaves and reject an accuracy-targeted candidate. Never copy the inner guarantee onto Hydra alone. +The code does not implement this yet: a Hydra state currently has its +per-subpopulation state's guarantee, without the shared-grid term. ## Adding or changing an accuracy target @@ -308,23 +315,27 @@ optional `accuracy_metric` and an `accuracy_scope`; built with `exact()` or 2. Define the parameters and evidence needed to prove satisfaction. 3. Add the satisfaction rule, including invalid, unknown, and unsupported behavior. -4. Add allocation behavior where the target can be divided across layers; do - not invent allocation for non-divisible semantics. +4. Add how the target passes through the error contracts down to the + readouts; do not invent a split for a target that does not divide. 5. Add serialization, DAG explanation, and compatibility behavior. -6. Add local and composition contracts for summaries that claim the target. +6. Add the readout guarantee for summaries that claim the target, and an + error contract for each operator its error must pass through. 7. Keep candidates fail-closed until the end-to-end guarantee path is implemented and tested. Do not coerce membership, distributional, or another non-epsilon requirement into an existing variant merely to reuse its API. -## Adding a summary or composition +## Adding a summary or an error contract -For a new summary, register its error metric and a local contract tied to the -same parameters that sizing commits. Record estimator/version premises in -provenance. For a new composition, specify compatible input and output metrics, -the bound rule, failure composition, required statistics, and unsupported -cases. Missing evidence must stay symbolic or produce a structured rejection. +For a new summary, give its readouts a guarantee per listed θ at a failure +probability (`guarantee/`) in one metric, tied to the parameters its list +commits. Record estimator/version premises in the guarantee module's docs. +For a new error contract, add the operator's row to `error_contract`: the +metric it keeps, the error factor per input and any limit up to which the +linear bound holds. An operator without a +row has no contract, so an approximate input never reaches it. Missing +evidence must stay symbolic or produce a structured rejection. Update DAG export whenever new guarantee or rejection data is introduced. @@ -332,8 +343,9 @@ Update DAG export whenever new guarantee or rejection data is introduced. Tests must cover: -- exact, additive, relative, Lipschitz, sum, and extremum propagation; -- incompatible metrics and unsupported composition; +- each operator's error contract per metric, and operators without one; +- the metric rule and the readout levels a target gives through the + contracts; - malformed and unknown symbolic values; - target checking after parameter clamps; - CountSketch L2 sizing distinct from CMS L1 sizing; diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 2cb7da985..87cab901f 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -55,13 +55,13 @@ estimator; an execution test in `crates/integration-tests/tests/` such as A new way for computations of one Pass 1 candidate to share, split or reuse work. -1. Add a memo rule under `crates/stage1-pass2-reuse-candidates/src/memo/rules/` - (for example `rollup.rs` or `topk.rs`): it adds candidates (and, when it +1. Add a memo rule under `crates/stage1-pass2-reuse-candidates/src/rules/` + (for example `rollup/` or `topk.rs`): it adds candidates (and, when it needs them, groups) to the memo groups it implements, keeping every existing candidate. Read summary families only through - `SummaryTypeCapability`. Shared rewrite helpers live in - `pass2/asap_aware_cse.rs`. -2. Declare it in `rules.rs` and call it from `generate_logical_asap_memo_candidates` + `SummaryTypeCapability`. The helpers rules add candidates with live in + `rules/candidates.rs`. +2. Declare it in `rules/mod.rs` and call it from `generate_logical_asap_memo_candidates` (`stage1-pass2-reuse-candidates/src/lib.rs`), after the existing rules. 3. Add its row to the Pass 2 rule table of the @@ -78,7 +78,7 @@ contains the candidate and that Stage 3 prices it. ## 3. Add a Stage 2 physical properties option 1. Add the `Lifetime` variant or the valid (timing, lifetime) pair - (`crates/types/src/stages/physical/properties.rs`, `PhysicalProperties::new` and + (`crates/types/src/stages/physical/properties/mod.rs`, `PhysicalProperties::new` and `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. 2. Offer it in `crates/stage2-physical-optimization/src/lib.rs` (the diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 070c018ce..8a7b4f3b0 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -32,7 +32,6 @@ pub struct OperatorNode { pub operator: Operator, // NonASAP(NonASAPOp) | ASAP(ASAPOp) pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar pub schema: Schema, // output schema, derived at construction - pub guarantee: Option, // None until accuracy assessment establishes one } ``` @@ -44,7 +43,6 @@ pub struct OperatorNode { - `schema` is derived by `OperatorNode::new(operator)`; it fails when the schema cannot be derived (a column reference out of range, a reserved ASAP operator). ASAP planning may retain a more specific schema through `OperatorNode::with_schema`. -- `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. - A node has no physical properties (timing, lifetime): a logical candidate has only its logical properties (#1200). Physical stages keep each node's `PhysicalProperties` in a `PhysicalPropertiesAssignment` keyed by node; `ir::timing::apply_physical_properties` @@ -148,7 +146,7 @@ yet — and calls `asap_frontend_common::resolve_root`, which does three things The result is `Rc`. `lower_sql` / `lower_sql_dialect` and `lower_metricsql` return it; `lower_promql_query_workload` returns a `QueryRoot` per query. -Workload search then runs structural CSE (Stage 0's `memo_of_queries` in `crates/stage0-memo/src/lib.rs`, hash-consing each operator into a `Memo` group through `Memo::find_or_add_group` and `crates/types/src/stages/logical/memo/hash_consing.rs`) once +Workload search then runs structural CSE (Stage 0's `memo_of_queries` in `crates/stage0-memo/src/lib.rs`, hash-consing each operator into a `Memo` group through `Memo::find_or_add_group` and `crates/types/src/stages/logical/memo/hash_consing/mod.rs`) once across every root: bottom-up hash-consing where the structural hash is only a filter and the typed `PartialEq` decides sharing, following scalar references like any other input, and gated by `NonASAPOp::is_shareable_when_equal` for non-ASAP producers. diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index 1f6564ca2..b59e5160c 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -49,7 +49,7 @@ or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. | `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | | Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_physical_properties` writes it from a `PhysicalPropertiesAssignment` (default: all query time) | | `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | -| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `memo_of_queries` (`asap_stage0_memo`, `crates/stage0-memo/src/lib.rs`), hash-consing through `Memo::find_or_add_group` (`crates/types/src/stages/logical/memo/hash_consing.rs`) | +| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `memo_of_queries` (`asap_stage0_memo`, `crates/stage0-memo/src/lib.rs`), hash-consing through `Memo::find_or_add_group` (`crates/types/src/stages/logical/memo/hash_consing/mod.rs`) | | `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::plan::physical_asap_dag::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | | Exported schema JSON `columns` | `fields` | diff --git a/docs/user_guide_docs/asap-planner.md b/docs/user_guide_docs/asap-planner.md index 3b29c71f9..f8c293b0f 100644 --- a/docs/user_guide_docs/asap-planner.md +++ b/docs/user_guide_docs/asap-planner.md @@ -62,7 +62,7 @@ under the stage that failed. | --- | --- | | `stage0` | The frontends' query DAGs and their memo groups. | | `stage1_pass1` | Stage 1 Pass 1: each computation's candidates, and the candidates pruned with their reason. | -| `stage1_pass2` | Stage 1 Pass 2: the memo after local window composition (a candidate for every legal tumbling-window width), then the memo after the sharing rules, which Stage 2 implements. | +| `stage1_pass2` | Stage 1 Pass 2: the memo after the reuse rules and chain groups (a chain group for every legal tumbling-window width), with the logical edges and the pruned candidates, which Stage 2 implements. | | `stage2` | Stage 2: the physical candidates and the ones pruned. | | `stage3` | Stage 3: the selected plan, its cost per node, each query's accepted accuracy and the search status. |