diff --git a/crates/accuracy/src/guarantee/cardinality.rs b/crates/accuracy/src/guarantee/cardinality.rs index a437da6de..5e03bd0b6 100644 --- a/crates/accuracy/src/guarantee/cardinality.rs +++ b/crates/accuracy/src/guarantee/cardinality.rs @@ -33,6 +33,7 @@ use super::*; /// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. const KMV_MIN_K: u32 = 2; /// The largest `k` sizing returns. +#[cfg(test)] const KMV_MAX_K: u32 = 1 << 26; pub(super) fn guarantee( @@ -81,6 +82,7 @@ pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { /// The smallest `k` whose KMV failure probability at relative error `eps` /// is at most `delta` ([`kmv_failure_probability`]), up to `2^26`; a /// degenerate `eps` or `delta` saturates to `2^26`. +#[cfg(test)] pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return KMV_MAX_K; diff --git a/crates/accuracy/src/guarantee/cms.rs b/crates/accuracy/src/guarantee/cms.rs index 6895ac6e1..ad48f58c2 100644 --- a/crates/accuracy/src/guarantee/cms.rs +++ b/crates/accuracy/src/guarantee/cms.rs @@ -25,6 +25,7 @@ pub(super) fn guarantee( } /// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. +#[cfg(test)] pub(crate) fn cms_width(eps: f64) -> u32 { saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) } diff --git a/crates/accuracy/src/guarantee/count_sketch.rs b/crates/accuracy/src/guarantee/count_sketch.rs index 4f72dfd99..52d6e4b7c 100644 --- a/crates/accuracy/src/guarantee/count_sketch.rs +++ b/crates/accuracy/src/guarantee/count_sketch.rs @@ -57,12 +57,14 @@ fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { } /// CountSketch `L2` point-query width: ε = sqrt(3/w). +#[cfg(test)] pub(crate) fn count_sketch_width(eps: f64) -> u32 { saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) } /// The smallest positive odd depth whose binomial median failure /// probability at `width` is at most `delta`, capped at 255. +#[cfg(test)] pub(crate) fn count_sketch_depth(width: u32, delta: f64) -> u32 { if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { return 255; diff --git a/crates/accuracy/src/guarantee/ddsketch.rs b/crates/accuracy/src/guarantee/ddsketch.rs index 9cde75477..5f1a7310c 100644 --- a/crates/accuracy/src/guarantee/ddsketch.rs +++ b/crates/accuracy/src/guarantee/ddsketch.rs @@ -20,6 +20,7 @@ pub(super) fn guarantee( )) } +#[cfg(test)] pub(super) fn size_params(epsilon: f64) -> SketchParams { SketchParams::DDSketch { alpha: epsilon } } diff --git a/crates/accuracy/src/guarantee/hll.rs b/crates/accuracy/src/guarantee/hll.rs index 7da74066d..ea463ff96 100644 --- a/crates/accuracy/src/guarantee/hll.rs +++ b/crates/accuracy/src/guarantee/hll.rs @@ -255,6 +255,7 @@ pub fn min_precision( /// bound up to `⌈1/ε⌉` (18 when none up to 18 does). Pass 1 then raises /// the precision by the same rule ([`min_precision`]) when the source's own /// bound or a smaller failure probability needs it. +#[cfg(test)] pub(crate) fn hll_precision(eps: f64, delta: f64) -> u8 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return 18; diff --git a/crates/accuracy/src/guarantee/kll.rs b/crates/accuracy/src/guarantee/kll.rs index bc6f0f010..c450d4573 100644 --- a/crates/accuracy/src/guarantee/kll.rs +++ b/crates/accuracy/src/guarantee/kll.rs @@ -52,6 +52,7 @@ pub(crate) fn kll_rank_error_99(k: u32) -> f64 { /// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. Capped at /// [`KLL_MAX_K`], the largest `k` `asap_sketchlib` builds; an `eps` that /// needs more gets that `k`, whose guarantee then misses the target. +#[cfg(test)] pub(crate) fn kll_k(eps: f64) -> u32 { saturating_ceil( (KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / KLL_RANK_ERROR_EXPONENT_99), @@ -133,6 +134,7 @@ fn normal_quantile(p: f64) -> f64 { /// `k` for rank error `eps` at failure probability `delta`: the empirical /// 99% calibration for `delta >= 0.01`; below, the calibration at /// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. +#[cfg(test)] pub(crate) fn size_params(eps: f64, delta: f64) -> SketchParams { if delta.is_nan() || delta <= 0.0 || delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { return SketchParams::Kll { k: kll_k(eps) }; diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 9769da79f..fed1e51d9 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -7,6 +7,7 @@ use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; #[cfg(test)] use crate::requirements::meets; +#[cfg(test)] use asap_types::ir::operator::AggIntent; use asap_types::summary_type::SummaryTypeCapability; @@ -87,7 +88,10 @@ pub fn theoretical_guarantee( } } -#[allow(dead_code)] // Read again for the predefined θ lists (`parameter_list`), in #1207 PR 1. +/// Today's sizing: the parameters of algorithm `kind` for error `eps` at failure +/// probability `delta`. Test-only: the predefined lists (`parameter_list`) +/// are this sizing over an error grid, which a test keeps equal. +#[cfg(test)] pub(crate) fn size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -192,6 +196,7 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. +#[cfg(test)] pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { u32::try_from(k) .unwrap_or(u32::MAX) @@ -211,6 +216,108 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { mod tests { use super::*; + /// Each algorithm's predefined list (`parameter_list`) is today's sizing + /// over the error grid ε = 0.1 … 0.001 at failure probability 0.01, + /// smallest first, without repeats; KMV's and UnivMon's distinct counts + /// stop at ε = 0.01. Count-Min lists each sized width with depths 5, 7, + /// 10 and 12, Count Sketch with 47, 81, 117 and 155, and a heap variant + /// each of those with + /// every heap size of the 1-2-5 series 10 … 10 000. UnivMon's list also + /// combines each L2 shape with each distinct-count shape (28 entries). + #[test] + fn parameter_list_is_the_sizing_over_the_error_grid() { + use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; + use asap_types::summary_type::parameter_list; + let full_grid = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; + let distinct_grid = [0.1, 0.05, 0.02, 0.01]; + let cms_depths = [5, 7, 10, 12]; + let count_sketch_depths = [47, 81, 117, 155]; + let heap_sizes = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; + let l2 = AggIntent::FrequencyL2 { col: None }; + let top_1 = AggIntent::TopK { k: 1 }; + let quantile = default_quantile(0.5); + let distinct = default_cardinality(); + for (algorithm, intents) in [ + (SketchAlgorithm::Kll, vec![&quantile]), + (SketchAlgorithm::DDSketch, vec![&quantile]), + (SketchAlgorithm::Hll, vec![&distinct]), + (SketchAlgorithm::Kmv, vec![&distinct]), + (SketchAlgorithm::Cms, vec![&distinct]), + (SketchAlgorithm::CountSketch, vec![&distinct]), + (SketchAlgorithm::CmsWithHeap, vec![&top_1]), + (SketchAlgorithm::CountSketchWithHeap, vec![&top_1]), + (SketchAlgorithm::UnivMon, vec![&l2, &distinct, &quantile]), + ] { + let mut sized: Vec = Vec::new(); + for intent in intents { + let grid: &[f64] = match (algorithm, intent) { + ( + SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon, + AggIntent::Cardinality { .. }, + ) => &distinct_grid, + _ => &full_grid, + }; + for &eps in grid { + let listed: Vec = match size_params(algorithm, intent, eps, 0.01) + { + SketchParams::Cms { width, .. } => cms_depths + .iter() + .map(|&depth| SketchParams::Cms { width, depth }) + .collect(), + SketchParams::CountSketch { width, .. } => count_sketch_depths + .iter() + .map(|&depth| SketchParams::CountSketch { width, depth }) + .collect(), + SketchParams::CmsWithHeap { width, .. } => cms_depths + .iter() + .flat_map(|&depth| { + heap_sizes + .iter() + .map(move |&heap_size| SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }) + }) + .collect(), + SketchParams::CountSketchWithHeap { width, .. } => count_sketch_depths + .iter() + .flat_map(|&depth| { + heap_sizes.iter().map(move |&heap_size| { + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } + }) + }) + .collect(), + other => vec![other], + }; + for params in listed { + if !sized.contains(¶ms) { + sized.push(params); + } + } + } + } + let list = parameter_list(algorithm); + let (listed_sized, combined): (Vec<_>, Vec<_>) = + list.iter().partition(|params| sized.contains(params)); + assert_eq!( + format!("{listed_sized:?}"), + format!("{:?}", sized.iter().collect::>()), + "{algorithm:?}" + ); + let expected_combined = if algorithm == SketchAlgorithm::UnivMon { + 28 + } else { + 0 + }; + assert_eq!(combined.len(), expected_combined, "{algorithm:?}"); + } + } + /// Classic HLL's precision grows until its failure probability over the /// source's distinct bound meets the target, up to 18. #[test] diff --git a/crates/accuracy/src/guarantee/univmon.rs b/crates/accuracy/src/guarantee/univmon.rs index 2bf1d8a56..c0545c20c 100644 --- a/crates/accuracy/src/guarantee/univmon.rs +++ b/crates/accuracy/src/guarantee/univmon.rs @@ -17,18 +17,22 @@ //! (OctoSketch aggregators fed promoted deltas) are not covered: the //! executor rejects them. use super::*; +#[cfg(test)] use asap_types::ir::operator::AggIntent; /// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and /// seed 19 picks the layers a key reaches. +#[cfg(test)] pub(crate) const MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; /// The most rows an L2 sketch is sized with: the deepest odd shape within /// the 20 rows the executor builds ([`hash_independent`]), so the row median /// is one row's estimate, and within the hash budget at the more than 20 /// columns an L2 guarantee needs (5 column bits: 19 · 6 = 114 ≤ 128). +#[cfg(test)] const MAX_L2_ROWS: u32 = 19; /// Each row of a layer reads its column and sign bits from one hash. +#[cfg(test)] const HASH_BITS: u32 = 128; /// The per-row failure probability the L2 guarantee fixes: a row misses by /// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it @@ -39,6 +43,7 @@ const L2_ROW_FAILURE: f64 = 0.1; const DISTINCT_VARIANCE: f64 = 1.35; /// The smallest heap the distinct-count variance bound holds for. const DISTINCT_MIN_HEAP: u32 = 8; +#[cfg(test)] const CARD_CAP: u32 = 1 << 26; const L2_CONTRACT: &str = "univmon_layer0_f2_median_chebyshev_binomial_v1"; @@ -50,6 +55,7 @@ const MAX_DISTINCT: &str = "max_distinct"; /// The shape baseline for readouts without a size model: the total (exact /// whatever the shape) and entropy (uncertified). +#[cfg(test)] const BASELINE: SketchParams = SketchParams::UnivMon { heap_size: 256, sketch_rows: 5, @@ -147,6 +153,7 @@ fn l2_failure_probability(rows: u32) -> f64 { /// clamped to the hash budget: with more rows than [`MAX_L2_ROWS`], or more /// columns than `rows` leave bits for, the shape is the largest one built /// and its guarantee misses the target. +#[cfg(test)] fn l2_shape(eps: f64, delta: f64) -> (u32, u32) { let rows = (1..=MAX_L2_ROWS) .find(|&rows| l2_failure_probability(rows) <= delta) @@ -186,6 +193,7 @@ fn incomplete_top_layer_bound(heap_size: u32) -> f64 { /// The Chebyshev share of the distinct count's failure probability at /// heap `k` and relative error `eps`: `1.35/(k ε²)`. +#[cfg(test)] fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { DISTINCT_VARIANCE / (f64::from(heap_size) * eps * eps) } @@ -194,6 +202,7 @@ fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { /// `eps · F0` with probability at most `delta`, top-layer completeness /// included: `1.35/(k ε²) + (e/4)^(k/2) ≤ δ`. Capped; a capped heap's /// guarantee misses the target. +#[cfg(test)] fn distinct_heap(eps: f64, delta: f64) -> u32 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return CARD_CAP; @@ -320,6 +329,7 @@ pub(crate) fn distinct_guarantee( /// counter; /// - total and entropy: the baseline shape (the total is exact; entropy /// has no certified bound). +#[cfg(test)] pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { match intent { AggIntent::FrequencyL2 { .. } => { diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index c50268744..41fd07d4a 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -11,7 +11,6 @@ use asap_types::accuracy::{ ProbabilityExpr, }; use asap_types::ir::operator::arguments::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::merged_chain; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; @@ -528,32 +527,31 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> } /// The guarantee of an estimate or finalized value read from merged -/// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range (the whole chain's, or -/// its newest tumbling windows') when the -/// family's merged error for that read is exact or the same as that build's -/// (the family-capability table); none when it is approximate (evidence +/// tumbling windows (a state whose [`OperatorNode::summary_build`] is a +/// chain): the guarantee of the same read of one build over the time range +/// the merged windows cover when the +/// summary type's merged error for that read is exact or the same as that build's +/// (the summary-type capability table); none when it is approximate (evidence /// only, for plan selection to judge) or the family has no merge. `None` for /// any node that does not read merged tumbling windows. fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let merged = match node.asap()? { + let (state, query) = match node.asap()? { ASAPOp::SummaryEstimate { summary_input, query, - } => { - let chain = merged_chain(summary_input)?; - let family = chain.asap()?.produced_state()?; - SummaryTypeCapability::of(family).merged_error(query) - } - ASAPOp::FinalizeExactAccumulator { child } => { - let chain = merged_chain(child)?; - let family = chain.asap()?.produced_state()?; - // An exact accumulator's merged state is exact. - let merged = SummaryTypeCapability::of(family).merge_inputs; - merged.map(|_| MergedError::Exact) - } + } => (summary_input, Some(query)), + ASAPOp::FinalizeExactAccumulator { child } => (child, None), _ => return None, }; + let ASAPOp::SummaryTumblingWindowChain { summary_type, .. } = state.summary_build()? else { + return None; + }; + let capability = SummaryTypeCapability::of(&summary_type); + let merged = match query { + Some(query) => capability.merged_error(query), + // An exact accumulator's merged state is exact. + None => capability.merge_inputs.map(|_| MergedError::Exact), + }; if !matches!( merged, Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) @@ -663,40 +661,43 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match merged_chain(summary_input) - .unwrap_or_else(|| Rc::clone(summary_input)) - .asap()? - { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. + }) if (0.0..=1.0).contains(q) => { + let build = match summary_input.summary_build()? { + chain @ ASAPOp::SummaryTumblingWindowChain { .. } => chain, + _ => summary_input.asap()?.clone(), + }; + match &build { + ASAPOp::SummaryAgg { + input: + SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + .. + }, + filter: None, + child, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + input: + SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + .. + }, + filter: None, + child, + .. + } => estimates_raw_data_samples_of(child), + _ => None, } - | ASAPOp::SummaryTumblingWindowChain { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - }, + } Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { match summary_input.asap()? { ASAPOp::SummaryAgg { diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index 25e5c2410..ddb42ffea 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -1,9 +1,10 @@ -//! Sizing: the statistic a summary is read for, and the default sketch -//! parameters for an `(ε, δ)`. +//! The statistic a summary is read for. (Today's sizing for an `(ε, δ)` is +//! test-only, `estimators::size_params`: the predefined lists are checked +//! against it.) use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::ColumnRef; -use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::summary_type::SketchStatistic; /// The statistic a summary is read for to answer `intent`, or `None` when /// no sketch statistic serves the intent. @@ -22,47 +23,31 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { }) } -/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved -/// `(eps, delta)` accuracy budget. -/// -/// Each formula inverts the sketch family's standard error bound to the -/// smallest parameter satisfying the target, clamped to the family's sane -/// range. A non-positive ε saturates to the clamp maximum (tightest -/// allowed). -#[allow(dead_code)] // Read by `test_support` until #1207 PR 1 replaces it with `parameter_list`. -pub(crate) fn default_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - crate::guarantee::size_params(kind, intent, eps, delta) -} - #[cfg(test)] mod tests { use super::*; + use asap_types::summary_type::{SketchAlgorithm, SketchParams}; /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. #[test] fn epsilon_delta_sizes_cms_depth() { let intent = AggIntent::Count; assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), + crate::guarantee::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), SketchParams::Cms { width: 2719, depth: 7 } // ⌈e/0.001⌉, ⌈ln 1000⌉ ); assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), + crate::guarantee::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), SketchParams::Cms { width: 2719, depth: 5 } ); assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), + crate::guarantee::size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), SketchParams::Cms { width: 272, depth: 5 diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs index 67de5f443..bb1ba7986 100644 --- a/crates/accuracy/src/test_support.rs +++ b/crates/accuracy/src/test_support.rs @@ -6,10 +6,9 @@ use std::rc::Rc; use asap_types::accuracy::AccuracyGuarantee; use asap_types::ir::operator::arguments::Reduction; -use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{OperatorNode, ScalarExpr}; -use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate}; +use asap_types::summary_type::{SketchStatistic, SummaryUpdate}; use crate::composition::StreamNorm; use crate::propagation; @@ -70,11 +69,6 @@ pub fn source_distinct_guarantee( propagation::source_distinct_guarantee(family, query, input) } -/// `crate::guarantee::cms::cms_width`. -pub fn cms_width(eps: f64) -> u32 { - crate::guarantee::cms::cms_width(eps) -} - /// `crate::guarantee::cms::cms_depth`. pub fn cms_depth(delta: f64) -> u32 { crate::guarantee::cms::cms_depth(delta) @@ -85,16 +79,6 @@ pub fn kll_rank_error_99(k: u32) -> f64 { crate::guarantee::kll::kll_rank_error_99(k) } -/// `crate::sizing::default_size_params`. -pub fn default_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - crate::sizing::default_size_params(kind, intent, eps, delta) -} - /// `crate::composition::QUOTIENT_OPERAND_DOMAINS`. pub const QUOTIENT_OPERAND_DOMAINS: &str = crate::composition::QUOTIENT_OPERAND_DOMAINS; diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 9b2dc957f..fc7f44900 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -8,6 +8,8 @@ use serde::{Deserialize, Serialize}; use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::non_asap::NonASAPOp; +use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::arguments::Reduction; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; @@ -76,8 +78,9 @@ pub enum ASAPOp> { /// `reduction` and `filter`, built over each window /// `(k·w, (k+1)·w]` of `child`, `w` = `tumbling_window_width_ms`, on a /// grid anchored at the epoch. Each window is built when it ends. Output: - /// one row per window, (window end, keys, state), a state with a time - /// index. It has no range and no window count: a `TimeRange` (and + /// one row per window and key, (window_end, keys, state), a state with + /// a time index (`window_end`, column 0), unique by (window_end, keys). + /// It has no range and no window count: a `TimeRange` (and /// `TimeShift`) over it selects the windows a query reads, and how many /// windows it retains is $h_v$, a Stage 3 choice. // Written as `PaneChain` before #1163; the aliases read those documents. @@ -282,13 +285,13 @@ impl ASAPOp { summary_type: family, reduction, .. - } - | SummaryTumblingWindowChain { + } => summary_agg_schema(child.schema(), family, reduction)?, + SummaryTumblingWindowChain { child, summary_type: family, reduction, .. - } => summary_agg_schema(child.schema(), family, reduction)?, + } => chain_schema(child.schema(), family, reduction)?, SummaryEstimate { summary_input, query, @@ -337,9 +340,8 @@ impl ASAPOp { if let Some((build, schema)) = rolled_up_build(child) { return finalized_schema(Some(&build), &schema); } - if super::tumbling_window_chain::merged_chain(child).is_some() { - todo!("#1207 PR 1: finalize the windows a TimeRange over a chain selects as one build over that TimeRange") - } + // The windows a `TimeRange` over a chain selects, merged, + // finalize as the chain's build over that range. finalized_schema(summary_build(child).as_ref(), child.schema())? } MaintainPopulation { child, .. } => child.schema().clone(), @@ -411,9 +413,22 @@ impl ASAPOp { } => { self.validate_inputs()?; let input = children[0].schema(); - match group_by { - None => input.clone(), - Some(keys) => regrouped_schema(input, keys)?, + match (group_by, summary_build(&children[0])) { + (Some(keys), _) => regrouped_schema(input, keys)?, + // The windows a `TimeRange` over a chain selects merge + // into the state of the chain's build over that range, + // which has no window column (a `TimeRange` and a + // `TimeShift` keep their input's schema). + ( + None, + Some(SummaryTumblingWindowChain { + child, + summary_type, + reduction, + .. + }), + ) => summary_agg_schema(child.schema(), &summary_type, &reduction)?, + (None, _) => input.clone(), } } SummarySubtract { .. } @@ -475,6 +490,19 @@ impl ASAPOp { "summary merge requires exactly one state column".into(), )); } + // Over a time-indexed state: the windows one `TimeRange` + // over a chain selects, not the whole chain. + if children.iter().any(is_time_indexed_state) + && !matches!( + children.as_slice(), + [child] if matches!(child.operator(), Operator::NonASAP(NonASAPOp::TimeRange { .. })) + ) + { + return Err(SchemaDerivationError::InvalidScalarSignature( + "a summary merge over a time-indexed state reads one TimeRange over a chain" + .into(), + )); + } for child in children { needs_state(child, "SummaryMerge")?; if child.schema() != first.schema() { @@ -656,22 +684,31 @@ fn rolled_up_build( return None; }; let input = selected_from(input).unwrap_or_else(|| input.clone()); - let build = match input.operator() { - Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => { + // A build, or the windows a `TimeRange` over a chain selects, merged: + // the chain's build over that range, whose schema is the build's over + // the chain's input. + let build = match (input.operator(), summary_build(&input)) { + (Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }), _) + | (_, Some(op @ ASAPOp::SummaryTumblingWindowChain { .. })) => { op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) } - _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { - todo!("#1207 PR 1: roll up the windows a TimeRange over a chain selects as one build over that TimeRange") - } _ => return None, }; - let ASAPOp::SummaryAgg { + let (ASAPOp::SummaryAgg { + child, + summary_type, + input: update, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { child, - summary_type: family, + summary_type, input: update, reduction, filter, - } = build + .. + }) = build else { return None; }; @@ -684,11 +721,11 @@ fn rolled_up_build( }) .collect::>>()?; let reduction = Reduction::by(mapped); - let schema = summary_agg_schema(child.schema(), &family, &reduction).ok()?; + let schema = summary_agg_schema(child.schema(), &summary_type, &reduction).ok()?; Some(( ASAPOp::SummaryAgg { child, - summary_type: family, + summary_type, input: update, reduction, filter, @@ -723,6 +760,39 @@ pub(crate) fn summary_agg_schema( Ok(schema) } +/// The schema of a `SummaryTumblingWindowChain` of `summary_type` by `reduction` +/// over rows of schema `child`: a state with a time index, one row per +/// window and key, (window_end, keys…, state). `window_end` (column 0, the +/// time index) is computed, so it takes no name or table from `child`; the +/// build's own time column, which a per-entity build keeps, is dropped. The +/// unique key is (window_end, keys). +fn chain_schema( + child: &Schema, + summary_type: &FieldDataType, + reduction: &Reduction, +) -> Result { + let mut schema = summary_agg_schema(child, summary_type, reduction)?; + let time = child + .time_index + .and_then(|index| child.fields.get(index)) + .ok_or_else(|| { + SchemaDerivationError::InvalidScalarSignature( + "a tumbling-window chain reads rows with a time column".into(), + ) + })?; + if let Some(build_time) = schema.time_index.take() { + schema.fields.remove(build_time); + } + schema + .fields + .insert(0, Field::new("window_end", time.dtype.clone(), false)); + schema.time_index = Some(0); + schema.unique_keys = vec![(0..schema.fields.len()) + .filter(|&column| schema.fields[column].is_plain()) + .collect()]; + Ok(schema) +} + /// The schema `FinalizeExactAccumulator` derives over state of /// `state_schema` that `build` (the state's `SummaryAgg`, if any) builds: /// each exact aggregate replaced by the value it finalizes to. @@ -730,14 +800,24 @@ fn finalized_schema( build: Option<&ASAPOp>, state_schema: &Schema, ) -> Result { - let value_result = if let Some(ASAPOp::SummaryAgg { - child: source, - summary_type: FieldDataType::ExactAggregate(kind, _), - input, - reduction, - filter, - .. - }) = build + let value_result = if let Some( + ASAPOp::SummaryAgg { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + }, + ) = build { { use crate::ir::operator::AggIntent; @@ -843,22 +923,24 @@ fn finalized_data_type(kind: &crate::summary_type::ExactKind) -> DataType { } } -/// The states `node` selects from when it is a `Filter` that keeps whole -/// summary states by their plain columns (its output is state); `None` for +/// The states `node` selects from when it keeps whole summary states (its +/// output is state): a `Filter` by their plain columns, or a `TimeRange` or +/// `TimeShift` over a time-indexed state, which selects windows. `None` for /// any other input. pub fn selected_from(node: &I) -> Option { match node.operator() { - Operator::NonASAP(crate::ir::NonASAPOp::Filter { child, .. }) - if node.result_kind() == OperatorResultKind::State => - { - Some(child) - } + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. }, + ) if node.result_kind() == OperatorResultKind::State => Some(child), _ => None, } } /// The build whose state this summary state carries: a `SummaryAgg` or -/// `SummaryTumblingWindowChain` itself, or the first input's build under +/// `SummaryTumblingWindowChain` itself, the chain under the `TimeRange` and +/// `TimeShift`s that select its windows, or the first input's build under /// `SummaryMerge`s (merged inputs have one definition, which summary /// coverage checks). `None` under a regrouping merge, whose states no build /// carries (its coverage's definition is the build it stands for), and for @@ -873,6 +955,22 @@ pub fn summary_build(state: &I) -> Option> { group_by: None, .. }) => summary_build(children.first()?), + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => summary_build(&child), + _ => None, + } +} + +/// The summary type a build operator builds: a `SummaryAgg`'s or a +/// `SummaryTumblingWindowChain`'s `summary_type`; `None` for every other +/// operator. +pub fn summary_type_of_operator(operator: &Operator) -> Option<&FieldDataType> { + match operator { + Operator::ASAP( + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, + ) => Some(summary_type), _ => None, } } diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 7eb6bbfba..2312fd73c 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use super::input::OperatorInput; use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::agg_intent::AggIntent; use crate::ir::operator::arguments::{ BinaryOpKind, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, @@ -921,6 +922,12 @@ impl NonASAPOp { TimeRange { child, .. } if child.result_kind() == OperatorResultKind::Relation => { OperatorResultKind::Relation } + // Over a time-indexed state (a chain): the windows it selects. + TimeRange { child, .. } | TimeShift { child, .. } + if child.result_kind() == OperatorResultKind::State => + { + OperatorResultKind::State + } TimeRange { kind, .. } => match kind { TimeRangeKind::Instant => OperatorResultKind::InstantVector, TimeRangeKind::Range => OperatorResultKind::RangeVector, @@ -1162,6 +1169,24 @@ impl NonASAPOp { return Err(invalid("label expression requires a string")); } } + TimeRange { child, .. } | TimeShift { child, .. } + if child.result_kind() == OperatorResultKind::State => + { + if !is_time_indexed_state(child) { + return Err(invalid( + "a time selection over state requires a time-indexed state", + )); + } + if matches!( + self, + TimeRange { + kind: TimeRangeKind::Instant, + .. + } + ) { + return Err(invalid("a time selection over a chain is a range")); + } + } TimeRange { child, .. } => { if child.result_kind() != OperatorResultKind::Relation { instant(child)?; diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index aca6285b0..065a3a310 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -1,33 +1,23 @@ //! Tumbling-window chains ([`ASAPOp::SummaryTumblingWindowChain`]): one //! row per window of width `w`, on a grid anchored at the epoch, a state //! with a time index. A chain has no range and no window count: a -//! `TimeRange` (and `TimeShift`) over it selects the windows a query reads -//! (#1207 PR 1), and how many windows it retains is $h_v$ (#1207 PR 7). +//! `TimeRange` (and `TimeShift`s) over it select the windows a query reads, +//! a `SummaryMerge` over that selection is the state of the chain's build +//! over the same range, and how many windows it retains is $h_v$ (#1207 +//! PR 7). use super::asap::ASAPOp; use super::input::OperatorInput; use super::node::Operator; +use super::non_asap::NonASAPOp; -/// The `SummaryTumblingWindowChain` a summary state carries merged: the -/// chain itself, or the one chain a `SummaryMerge` that keeps its groups -/// reads. `None` for any other input, a rollup (regrouping merge) included. -pub fn merged_chain(state: &I) -> Option { +/// Whether `state` is a state with a time index: a chain, or a `TimeRange` +/// or `TimeShift` over one. +pub(crate) fn is_time_indexed_state(state: &I) -> bool { match state.operator() { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), - Operator::ASAP(ASAPOp::SummaryMerge { - children, - group_by: None, - .. - }) => match children.as_slice() { - [child] - if matches!( - child.operator(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) => - { - Some(child.clone()) - } - _ => None, - }, - _ => None, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => true, + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => is_time_indexed_state(&child), + _ => false, } } diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs index 61d59a103..618c228bc 100644 --- a/crates/types/src/stages/logical/candidate.rs +++ b/crates/types/src/stages/logical/candidate.rs @@ -2,10 +2,12 @@ //! (concepts doc `memo-groups-properties-requirements.md`, section 3; //! #1203). +use crate::ir::operator::asap::summary_type_of_operator; +use crate::ir::schema::FieldDataType; use crate::ir::Operator; use crate::ir::SchemaDerivationError; use crate::stages::{LogicalRequirements, MemoGroupId}; -use crate::summary_type::SketchParams; +use crate::summary_type::{parameter_list, SketchParams}; /// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). @@ -90,8 +92,21 @@ impl LogicalProperties { expression: &CandidateExpression, parameters: Option, ) -> Result { - let _ = (expression, parameters); - todo!("#1207 PR 1: reject invalid parameters") + let algorithm = match summary_type_of_operator(&expression.operator) { + Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), + _ => None, + }; + match (algorithm, ¶meters) { + (Some(_), None) => return Err(LogicalPropertiesError::SketchBuildWithoutParameters), + (None, Some(_)) => return Err(LogicalPropertiesError::ParametersOnExactOperator), + (Some(algorithm), Some(parameters)) => { + if !parameter_list(algorithm).contains(parameters) { + return Err(LogicalPropertiesError::ParametersNotInList); + } + } + (None, None) => {} + } + Ok(Self { parameters }) } /// $\boldsymbol\theta_v$. @@ -111,3 +126,153 @@ pub enum LogicalPropertiesError { #[error("the parameters are not in their algorithm's predefined list")] ParametersNotInList, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::ASAPOp; + use crate::summary_type::SketchAlgorithm; + + use crate::ir::operator::arguments::Reduction; + + use crate::ir::scalar::expr_ir::ColumnRef; + + use crate::ir::schema::FieldDataType; + use crate::summary_type::{ExactKind, HydraParams, SummaryUpdate}; + + /// `SummaryAgg` over group 0 building a state of `summary_type`. + fn summary_agg(summary_type: FieldDataType) -> CandidateExpression { + CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryAgg { + child: MemoGroupId(0), + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + } + } + + const KLL_269: SketchParams = SketchParams::Kll { k: 269 }; + + const CMS: SketchParams = SketchParams::Cms { + width: 272, + depth: 5, + }; + + const HYDRA_CMS: SketchParams = SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 1_024, + }); + + /// `LogicalProperties::new` over every operator kind and parameters: a + /// sketch build (a chain included) needs parameters of its algorithm from + /// its list; parameters are rejected on an operator that builds no + /// sketch. + #[test] + fn logical_properties_new_over_every_operator_and_parameters() { + use LogicalPropertiesError as Invalid; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let kll_chain = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }), + }; + let readout = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: MemoGroupId(0), + query: crate::summary_type::SketchStatistic::Quantile { q: 0.5 }, + }), + }; + // k = 270 is KLL's structure but not in its list. + let parameters = [ + None, + Some(KLL_269), + Some(CMS), + Some(HYDRA_CMS), + Some(SketchParams::Kll { k: 270 }), + ]; + let no_parameters = Err(Invalid::ParametersOnExactOperator); + let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); + let not_in_list = Err(Invalid::ParametersNotInList); + let exact = [ + Ok(()), + no_parameters, + no_parameters, + no_parameters, + no_parameters, + ]; + // Another algorithm's parameters are not in KLL's list either. + let kll_build = [ + needs_parameters, + Ok(()), + not_in_list, + not_in_list, + not_in_list, + ]; + // Per operator: the result for each of `parameters`. + let table = [ + (summary_agg(kll), kll_build), + // No Hydra θ is listed yet, so no Hydra build is valid. + ( + summary_agg(FieldDataType::Sketch(SketchAlgorithm::HydraCms)), + [ + needs_parameters, + not_in_list, + not_in_list, + not_in_list, + not_in_list, + ], + ), + (summary_agg(FieldDataType::exact(ExactKind::Sum)), exact), + (kll_chain, kll_build), + (readout, exact), + ]; + for (expression, by_parameters) in table { + for (parameters, expected) in parameters.iter().zip(by_parameters) { + let properties = LogicalProperties::new(&expression, parameters.clone()); + assert_eq!( + properties.as_ref().map(|_| ()).map_err(|e| *e), + expected, + "{:?} {parameters:?}", + expression.operator.kind_name() + ); + if let Ok(properties) = properties { + assert_eq!(properties.parameters(), parameters.as_ref()); + } + } + } + } + + /// `summary_type_of_operator` reads the summary type of a `SummaryAgg` + /// and of a chain, and of no other operator. + #[test] + fn summary_type_of_operator_reads_the_builds_summary_type() { + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![MemoGroupId(0)], + group_by: None, + }); + assert_eq!( + summary_type_of_operator(&summary_agg(kll.clone()).operator), + Some(&kll) + ); + assert_eq!(summary_type_of_operator(&chain), Some(&kll)); + assert_eq!(summary_type_of_operator(&merge), None); + } +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index efef26be1..cdd7e212d 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -82,8 +82,7 @@ impl Memo { /// The candidate `id`. pub fn candidate(&self, id: CandidateForMemoGroupId) -> &CandidateForMemoGroup { - let _ = id; - todo!("#1207 PR 2") + &self.groups[id.group.0 as usize].candidates[id.index_in_group] } /// Replaces candidate `id`'s input requirements $r^{\log}_{c,i}$, one diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index 130817db2..c5891c112 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -2,8 +2,9 @@ //! valid $p^{\mathrm{phys}} = (\tau, \ell)$ (#1200). use crate::ir::properties::ExecutionTiming; -use crate::stages::PhysicalProperties; +use crate::ir::{ASAPOp, Operator}; use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; +use crate::stages::{Lifetime, PhysicalProperties}; /// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -20,24 +21,70 @@ pub struct PhysicalCandidate { impl PhysicalCandidate { /// Whether a consumer running in `context` can read this producer (the - /// concepts doc's alignment table, section 2). True for $\ell$ = None, - /// and for `OneEvaluation` in the same context. A chain producer - /// (`SummaryTumblingWindowChain`) with $\ell$ = `AcrossEvaluations`, with - /// $w$ the chain operator's width, needs every time of the context on - /// its grid: $w$ divides the context's period and phase. Any other - /// producer with $\ell$ = `AcrossEvaluations` is stored on its group's - /// evaluation grid (`evaluation_grid_of_group`): it delivers on every - /// context when the group does not depend on the evaluation time, else - /// when that grid holds every time of the context. In - /// `Materialized(v)`, the context's times are v's, read from - /// `candidates`. + /// concepts doc's alignment table, section 2): true for $\ell$ = None, + /// which runs in the consumer's own context; for `OneEvaluation` only in + /// an `Evaluation` context, the one evaluation its result belongs to. + /// With $\ell$ = `AcrossEvaluations`, every execution time of the + /// context must be a time the producer has stored: for a chain + /// (`SummaryTumblingWindowChain`) of width $w$, $w$ divides an + /// evaluation grid's period and phase, or a refresh window's width and + /// phase; any other result is stored on its group's evaluation grid + /// (`evaluation_grid_of_group`), which must hold every time of the + /// context, unless the group does not depend on the evaluation time + /// (then it delivers on every context). pub fn delivers_on( &self, candidates: &PhysicalASAPMemoCandidates, context: &ExecutionContext, ) -> bool { - let _ = (candidates, context); - todo!("#1207 PR 1: the alignment table of section 2") + match self.physical_properties.lifetime() { + Lifetime::None => true, + Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), + Lifetime::AcrossEvaluations(_) => { + let logical = &candidates.logical_candidates; + let group = self.logical_candidate_id.group; + let chain_width_ms = match logical + .memo + .candidate(self.logical_candidate_id) + .expression + .operator + { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(tumbling_window_width_ms), + _ => None, + }; + let depends_on_evaluation_time = logical.memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time; + if chain_width_ms.is_none() && !depends_on_evaluation_time { + return true; + } + let (period_ms, phase_ms) = match context { + ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), + ExecutionContext::IngestionPipeline(window) => { + (window.width_ms, window.phase_ms) + } + ExecutionContext::Materialized(_) => { + todo!( + "#1207 PR 7: the times of a Materialized(v) context, v's own schedule" + ) + } + }; + match chain_width_ms { + Some(width_ms) => { + period_ms.is_multiple_of(width_ms) && phase_ms.is_multiple_of(width_ms) + } + None => { + logical.evaluation_grid_of_group[group.0 as usize].is_some_and(|grid| { + period_ms.is_multiple_of(grid.period_ms) + && phase_ms % grid.period_ms == grid.phase_ms % grid.period_ms + }) + } + } + } + } } } @@ -48,6 +95,17 @@ pub fn producer_satisfies_physical_requirements( producer: &PhysicalCandidate, consumer_timing: ExecutionTiming, ) -> bool { - let _ = (producer, consumer_timing); - todo!("#1207 PR 1: the timing x lifetime table of section 2") + use ExecutionTiming::{IngestionTime, QueryTime}; + match ( + producer.physical_properties.timing(), + producer.physical_properties.lifetime(), + ) { + (QueryTime, _) => consumer_timing == QueryTime, + (IngestionTime, Lifetime::None) => consumer_timing == IngestionTime, + // Materialized at ingestion: read at ingestion time and at query time. + (IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by `PhysicalProperties::new`") + } + } } diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index f3b1ec719..0aaf77cd4 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -100,3 +100,323 @@ pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, } references } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::properties::ExecutionTiming; + use crate::ir::{ASAPOp, Operator}; + use crate::stages::{ + producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, + Lifetime, PhysicalProperties, + }; + + use crate::plan::materialized_view::RefreshWindow; + + use crate::ir::operator::arguments::{Reduction, Source}; + + use crate::ir::scalar::expr_ir::ColumnRef; + + use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; + use crate::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; + + use crate::ir::{NonASAPOp, TimeRangeKind}; + + use crate::stages::StorageMedium; + + use crate::stages::physical::execution_context::EvaluationGrid; + + use crate::stages::{ + CandidateExpression, CandidateForMemoGroup, LogicalASAPMemoCandidates, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, + }; + + use ExecutionTiming::{IngestionTime, QueryTime}; + + /// The KLL build over the scan: not a chain. + const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(1), + index_in_group: 0, + }; + + /// The chain of tumbling windows 5 s wide over the scan. + const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(2), + index_in_group: 0, + }; + + /// The chain of tumbling windows 10 s wide over the scan. + const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(3), + index_in_group: 0, + }; + + /// X = the scan's last 5 minutes (`m[5m]`): a result that depends on + /// the evaluation time and is not a chain. + const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(4), + index_in_group: 0, + }; + + /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains + /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last + /// 5 minutes, each with its own expression as its one candidate. Built + /// through `find_or_add_group` and `add_candidate` (#1207 PR 2). + fn memo() -> Memo { + let mut memo = Memo::default(); + let scan = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let chain = |tumbling_window_width_ms| { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms, + }) + }; + for operator in [build, chain(5_000), chain(10_000)] { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new( + &expression, + Some(SketchParams::Kll { k: 269 }), + ) + .unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(group, candidate).unwrap(); + } + let operator = Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(group, candidate).unwrap(); + memo + } + + /// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; + /// `delivers_on` reads nothing else. + fn candidates( + memo: Memo, + evaluation_grid_of_group: Vec>, + ) -> PhysicalASAPMemoCandidates { + PhysicalASAPMemoCandidates { + logical_candidates: LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group, + }, + physical_candidates: vec![], + pruned_candidates: vec![], + edges: vec![], + admissible_edges_into_input: HashMap::new(), + admissible_edges_out_of_candidate: vec![], + } + } + + fn candidate( + logical_candidate_id: CandidateForMemoGroupId, + timing: ExecutionTiming, + lifetime: Lifetime, + ) -> PhysicalCandidate { + PhysicalCandidate { + logical_candidate_id, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + } + } + + fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms, + }, + } + } + + fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) + } + + const MEMORY: StorageMedium = StorageMedium::Memory; + + /// A `None` producer runs in the consumer's context and delivers on + /// every one; a `OneEvaluation` producer only on an evaluation context, + /// the one evaluation its result belongs to. Neither reads the memo. + #[test] + fn delivers_on_by_lifetime_without_windows() { + let candidates = candidates(Memo::default(), vec![]); + let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); + let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); + let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); + for (producer, context, delivers) in [ + (&unmaterialized, evaluation(7_000, 3_000), true), + (&unmaterialized, ingestion(5_000, 0), true), + (&unmaterialized, materialized, true), + (&one_evaluation, evaluation(7_000, 3_000), true), + (&one_evaluation, ingestion(5_000, 0), false), + (&one_evaluation, materialized, false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} in {context:?}", + producer.physical_properties + ); + } + } + + /// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase + /// 0) delivers when $w$ divides the context's period (or refresh-window + /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s + /// serves both. + #[test] + #[ignore = "filled in by #1207 PR 2 (#1230)"] + fn delivers_on_a_chain_when_its_width_divides_the_context_times() { + let candidates = candidates(memo(), vec![None; 5]); + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let ingestion_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for (producer, context, delivers) in [ + // Evaluation grids: w divides period and phase. + (&across_5s, evaluation(10_000, 0), true), + (&across_5s, evaluation(10_000, 5_000), true), + (&across_5s, evaluation(10_000, 2_000), false), + (&across_5s, evaluation(7_000, 0), false), + (&ingestion_10s, evaluation(10_000, 0), true), + (&ingestion_10s, evaluation(10_000, 5_000), false), + // Refresh windows: w divides width and phase. + (&across_5s, ingestion(5_000, 0), true), + (&across_5s, ingestion(10_000, 5_000), true), + (&across_5s, ingestion(10_000, 2_500), false), + (&ingestion_10s, ingestion(5_000, 0), false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} {:?} in {context:?}", + producer.logical_candidate_id, + producer.physical_properties + ); + } + } + + /// A stored result that is not a chain delivers when its group's + /// evaluation grid holds every time of the context: X = + /// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is + /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or + /// another phase. Without a known grid it serves none. + #[test] + #[ignore = "filled in by #1207 PR 2 (#1230)"] + fn delivers_on_a_stored_result_on_its_group_grid() { + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); + let without_grid = candidates(memo(), vec![None; 5]); + let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + for (candidates, context, delivers) in [ + (&on_grid, evaluation(60_000, 0), true), + (&on_grid, evaluation(120_000, 0), true), + (&on_grid, evaluation(120_000, 60_000), true), + (&on_grid, evaluation(30_000, 0), false), + (&on_grid, evaluation(60_000, 15_000), false), + (&on_grid, ingestion(60_000, 0), true), + (&without_grid, evaluation(60_000, 0), false), + ] { + assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + } + } + + /// `producer_satisfies_physical_requirements` over every valid producer state and every valid + /// consumer state: a query-time producer serves only query-time + /// consumers, an ingestion-time producer that is not materialized only + /// ingestion-time consumers, and one materialized across evaluations + /// serves both. Chain widths do not matter: alignment is Stage 3's + /// per-context check. + #[test] + fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { + let states = |medium| { + [ + candidate(BUILD, QueryTime, Lifetime::None), + candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), + candidate(BUILD, IngestionTime, Lifetime::None), + candidate( + WINDOWS_5S, + IngestionTime, + Lifetime::AcrossEvaluations(medium), + ), + ] + }; + // Rows: producer state; columns: consumer state, in `states` order. + let expected = [ + [true, true, true, false, false], + [true, true, true, false, false], + [true, true, true, false, false], + [false, false, false, true, true], + [true, true, true, true, true], + ]; + for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (producer, row) in states(producer_medium).iter().zip(expected) { + for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { + assert_eq!( + producer_satisfies_physical_requirements( + producer, + consumer.physical_properties.timing() + ), + satisfies, + "{:?} -> {:?}", + producer.physical_properties, + consumer.physical_properties + ); + } + } + } + } + } +} diff --git a/crates/types/src/stages/physical/properties.rs b/crates/types/src/stages/physical/properties.rs index df1869ed4..3cd8f2a40 100644 --- a/crates/types/src/stages/physical/properties.rs +++ b/crates/types/src/stages/physical/properties.rs @@ -64,8 +64,12 @@ impl PhysicalProperties { timing: ExecutionTiming, lifetime: Lifetime, ) -> Result { - let _ = (timing, lifetime); - todo!("#1207 PR 1: reject the invalid combination of section 2") + match (timing, lifetime) { + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + } + _ => Ok(Self { timing, lifetime }), + } } /// $\tau_v$. @@ -89,32 +93,35 @@ mod tests { use super::*; use ExecutionTiming::{IngestionTime, QueryTime}; - const MEMORY: StorageMedium = StorageMedium::Memory; - - /// The one invalid combination of the concepts doc, section 2. - #[test] - fn new_rejects_one_evaluation_at_ingestion() { - assert_eq!( - PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), - Err(PhysicalPropertiesError::OneEvaluationAtIngestion) - ); - } - - /// The valid (τ, ℓ) pairs of section 2. + /// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for + /// both media, and keeps what it was given. #[test] - fn new_accepts_each_valid_pair() { - for (timing, lifetime) in [ - (QueryTime, Lifetime::None), - (IngestionTime, Lifetime::None), - (QueryTime, Lifetime::OneEvaluation(MEMORY)), - (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), - ( - IngestionTime, - Lifetime::AcrossEvaluations(StorageMedium::Disk), - ), - ] { - let properties = PhysicalProperties::new(timing, lifetime).unwrap(); - assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + fn new_over_every_timing_and_lifetime() { + for medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (timing, lifetime, valid) in [ + (QueryTime, Lifetime::None, true), + (QueryTime, Lifetime::OneEvaluation(medium), true), + (QueryTime, Lifetime::AcrossEvaluations(medium), true), + (IngestionTime, Lifetime::None, true), + (IngestionTime, Lifetime::OneEvaluation(medium), false), + (IngestionTime, Lifetime::AcrossEvaluations(medium), true), + ] { + let properties = PhysicalProperties::new(timing, lifetime); + match valid { + true => { + let properties = properties.unwrap(); + assert_eq!( + (properties.timing(), properties.lifetime()), + (timing, lifetime) + ); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + false => assert_eq!( + properties, + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ), + } + } } } } diff --git a/crates/types/src/summary_type/parameter_lists.rs b/crates/types/src/summary_type/parameter_lists.rs index c3ec86d4c..f87db5dbe 100644 --- a/crates/types/src/summary_type/parameter_lists.rs +++ b/crates/types/src/summary_type/parameter_lists.rs @@ -6,7 +6,302 @@ use super::{SketchAlgorithm, SketchParams}; /// The algorithm's predefined $\boldsymbol\theta$ list: structures only; the /// error of each, for a statistic at a failure probability, is its guarantee /// (`asap_accuracy::theoretical_guarantee`). +/// +/// The widths, `k`s, precisions, `alpha`s and heap sizes are today's sizing +/// (`asap_accuracy`'s test-only `size_params`) over the errors ε = 0.1, +/// 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, without +/// repeats; an `asap_accuracy` test keeps them equal. Distinct counts (KMV, +/// and UnivMon's distinct-count shapes) stop at ε = 0.01. Where δ is set by +/// a structural parameter, the list covers the δ levels with it: Count-Min +/// lists each width with each depth of [`CMS_DEPTHS`], Count Sketch with +/// each of [`COUNT_SKETCH_DEPTHS`]. A +/// UnivMon's list is its L2-norm shapes, its distinct-count shapes, each L2 +/// shape combined with each distinct-count shape, then its shape for every +/// other statistic. A heap variant combines each width × depth with each +/// heap size of [`HEAP_SIZES`]: a top-k needs heap_size ≥ k, and the next +/// listed step up is at most 2.5× too large. Empty for an algorithm without +/// an error model (the Hydra algorithms included), so no build of it is a +/// valid candidate. pub fn parameter_list(algorithm: SketchAlgorithm) -> &'static [SketchParams] { - let _ = algorithm; - todo!("#1207 PR 1: the predefined θ list per algorithm") + match algorithm { + SketchAlgorithm::Kll => &KLL_PARAMETERS, + SketchAlgorithm::DDSketch => &DDSKETCH_PARAMETERS, + SketchAlgorithm::Hll => &HLL_PARAMETERS, + SketchAlgorithm::Kmv => &KMV_PARAMETERS, + SketchAlgorithm::Cms => &CMS_PARAMETERS, + SketchAlgorithm::CountSketch => &COUNT_SKETCH_PARAMETERS, + SketchAlgorithm::CmsWithHeap => &CMS_WITH_HEAP_PARAMETERS, + SketchAlgorithm::CountSketchWithHeap => &COUNT_SKETCH_WITH_HEAP_PARAMETERS, + SketchAlgorithm::UnivMon => &UNIVMON_PARAMETERS, + // Listed, not `_`: an algorithm `asap_sketchlib` adds fails to + // compile here until it gets a list or is listed without one. + SketchAlgorithm::CountSketchWithL2Norm + | SketchAlgorithm::CmsWithSpaceSaving + | SketchAlgorithm::CountSketchWithSpaceSaving + | SketchAlgorithm::CmsFold + | SketchAlgorithm::CountSketchFold + | SketchAlgorithm::HllHip + | SketchAlgorithm::KllDynamic + | SketchAlgorithm::SpaceSaving + | SketchAlgorithm::Elastic + | SketchAlgorithm::Coco + | SketchAlgorithm::Bloom + | SketchAlgorithm::SetAggregator + | SketchAlgorithm::UnivMonTwoTier + | SketchAlgorithm::UnivMonQ + | SketchAlgorithm::HydraCms + | SketchAlgorithm::HydraCountSketch + | SketchAlgorithm::HydraKll + | SketchAlgorithm::HydraHll + | SketchAlgorithm::HydraUnivMon => &[], + } +} + +const KLL_PARAMETERS: [SketchParams; 7] = [ + SketchParams::Kll { k: 26 }, + SketchParams::Kll { k: 52 }, + SketchParams::Kll { k: 132 }, + SketchParams::Kll { k: 269 }, + SketchParams::Kll { k: 547 }, + SketchParams::Kll { k: 1_404 }, + SketchParams::Kll { k: 2_863 }, +]; + +const DDSKETCH_PARAMETERS: [SketchParams; 7] = [ + SketchParams::DDSketch { alpha: 0.1 }, + SketchParams::DDSketch { alpha: 0.05 }, + SketchParams::DDSketch { alpha: 0.02 }, + SketchParams::DDSketch { alpha: 0.01 }, + SketchParams::DDSketch { alpha: 0.005 }, + SketchParams::DDSketch { alpha: 0.002 }, + SketchParams::DDSketch { alpha: 0.001 }, +]; + +/// HLL's precision saturates at 18 from ε = 0.01 on. +const HLL_PARAMETERS: [SketchParams; 4] = [ + SketchParams::Hll { precision: 13 }, + SketchParams::Hll { precision: 15 }, + SketchParams::Hll { precision: 17 }, + SketchParams::Hll { precision: 18 }, +]; + +const KMV_PARAMETERS: [SketchParams; 4] = [ + SketchParams::Kmv { k: 1_118 }, + SketchParams::Kmv { k: 4_323 }, + SketchParams::Kmv { k: 26_656 }, + SketchParams::Kmv { k: 106_263 }, +]; + +/// The depths each Count-Min width is listed with: δ = e^−d ≤ 1e-2, 1e-3, +/// 1e-4 and 1e-5. +const CMS_DEPTHS: [u32; 4] = [5, 7, 10, 12]; +/// The depths each Count Sketch width is listed with, for the same δ +/// levels: Count Sketch reads the median of its rows, each wrong with +/// probability about 1/3, so δ = P(Binomial(d, 1/3) ≥ (d+1)/2), the +/// smallest odd d at or below each level. +const COUNT_SKETCH_DEPTHS: [u32; 4] = [47, 81, 117, 155]; + +const CMS_WIDTHS: [u32; 7] = [28, 55, 136, 272, 544, 1_360, 2_719]; +const COUNT_SKETCH_WIDTHS: [u32; 7] = [300, 1_200, 7_500, 30_000, 120_000, 750_000, 3_000_000]; + +const CMS_PARAMETERS: [SketchParams; 28] = with_depths( + &CMS_WIDTHS, + &CMS_DEPTHS, + SketchParams::Cms { width: 0, depth: 0 }, +); +const COUNT_SKETCH_PARAMETERS: [SketchParams; 28] = with_depths( + &COUNT_SKETCH_WIDTHS, + &COUNT_SKETCH_DEPTHS, + SketchParams::CountSketch { width: 0, depth: 0 }, +); + +/// Each of `widths` with each of `depths`, by width, then depth, as +/// `shape`'s variant (`Cms` or `CountSketch`). +const fn with_depths( + widths: &[u32; 7], + depths: &[u32; 4], + shape: SketchParams, +) -> [SketchParams; 28] { + const UNSET: SketchParams = SketchParams::Cms { width: 0, depth: 0 }; + let mut list = [UNSET; 28]; + let mut width = 0; + while width < widths.len() { + let mut depth = 0; + while depth < depths.len() { + let (width_value, depth_value) = (widths[width], depths[depth]); + list[width * depths.len() + depth] = match shape { + SketchParams::Cms { .. } => SketchParams::Cms { + width: width_value, + depth: depth_value, + }, + SketchParams::CountSketch { .. } => SketchParams::CountSketch { + width: width_value, + depth: depth_value, + }, + _ => panic!("a Count-Min or Count Sketch shape"), + }; + depth += 1; + } + width += 1; + } + list +} + +/// The heap sizes of a heap variant's list: the 1-2-5 series from 10 to +/// 10 000, so a top-k with k ≤ 10 000 has a listed heap. +const HEAP_SIZES: [u32; 10] = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; + +const CMS_WITH_HEAP_PARAMETERS: [SketchParams; 280] = with_heap_sizes(&CMS_PARAMETERS); +const COUNT_SKETCH_WITH_HEAP_PARAMETERS: [SketchParams; 280] = + with_heap_sizes(&COUNT_SKETCH_PARAMETERS); + +/// Each `Cms` (or `CountSketch`) shape of `shapes` with each heap size of +/// [`HEAP_SIZES`], as `CmsWithHeap` (or `CountSketchWithHeap`). +const fn with_heap_sizes(shapes: &[SketchParams; 28]) -> [SketchParams; 280] { + const UNSET: SketchParams = SketchParams::Cms { width: 0, depth: 0 }; + let mut list = [UNSET; 280]; + let mut shape = 0; + while shape < shapes.len() { + let mut heap = 0; + while heap < HEAP_SIZES.len() { + let heap_size = HEAP_SIZES[heap]; + list[shape * HEAP_SIZES.len() + heap] = match shapes[shape] { + SketchParams::Cms { width, depth } => SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + SketchParams::CountSketch { width, depth } => SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + }, + _ => panic!("a heap variant extends a Cms or CountSketch shape"), + }; + heap += 1; + } + shape += 1; + } + list +} + +/// UnivMon's L2-norm column counts and distinct-count heap sizes. +const UNIVMON_L2_COLUMNS: [u32; 7] = [624, 2_367, 14_349, 56_817, 226_130, 1_409_067, 5_630_630]; +const UNIVMON_DISTINCT_HEAP_SIZES: [u32; 4] = [13_500, 54_000, 337_500, 1_350_000]; + +const UNIVMON_PARAMETERS: [SketchParams; 40] = univmon_parameters(); + +/// The L2-norm shapes, the distinct-count shapes, then each L2 column +/// count × distinct heap size (by column count, then heap size), last the +/// shape for every other statistic. +const fn univmon_parameters() -> [SketchParams; 40] { + const fn univmon( + heap_size: u32, + sketch_rows: u32, + sketch_cols: u32, + layers: u8, + ) -> SketchParams { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + } + } + const BASELINE: SketchParams = univmon(256, 5, 1_024, 16); + let mut list = [BASELINE; 40]; + let mut next = 0; + let mut columns = 0; + while columns < UNIVMON_L2_COLUMNS.len() { + list[next] = univmon(1, 5, UNIVMON_L2_COLUMNS[columns], 1); + next += 1; + columns += 1; + } + let mut heap = 0; + while heap < UNIVMON_DISTINCT_HEAP_SIZES.len() { + list[next] = univmon(UNIVMON_DISTINCT_HEAP_SIZES[heap], 1, 1, 19); + next += 1; + heap += 1; + } + columns = 0; + while columns < UNIVMON_L2_COLUMNS.len() { + heap = 0; + while heap < UNIVMON_DISTINCT_HEAP_SIZES.len() { + list[next] = univmon( + UNIVMON_DISTINCT_HEAP_SIZES[heap], + 5, + UNIVMON_L2_COLUMNS[columns], + 19, + ); + next += 1; + heap += 1; + } + columns += 1; + } + list +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each predefined list holds one parameter type, without repeats, and + /// is empty exactly for the algorithms without an error model. Count-Min + /// lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, + /// 81, 117 and 155, and a + /// heap variant each of those with every heap size; UnivMon's list has 40 + /// entries. + #[test] + fn parameter_list_holds_its_algorithm_without_repeats() { + let listed = [ + SketchAlgorithm::Kll, + SketchAlgorithm::DDSketch, + SketchAlgorithm::Hll, + SketchAlgorithm::Kmv, + SketchAlgorithm::Cms, + SketchAlgorithm::CountSketch, + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + SketchAlgorithm::UnivMon, + ]; + for algorithm in listed.iter().copied().chain([ + SketchAlgorithm::HydraCms, + SketchAlgorithm::HydraCountSketch, + SketchAlgorithm::HydraKll, + SketchAlgorithm::HllHip, + ]) { + let list = parameter_list(algorithm); + assert_eq!( + list.is_empty(), + !listed.contains(&algorithm), + "{algorithm:?}" + ); + for (i, parameters) in list.iter().enumerate() { + assert_eq!( + std::mem::discriminant(parameters), + std::mem::discriminant(&list[0]) + ); + assert!(!list[..i].contains(parameters), "{parameters:?} repeats"); + } + } + let depths = |algorithm| -> Vec { + parameter_list(algorithm) + .iter() + .take(4) + .map(|parameters| match parameters { + SketchParams::Cms { depth, .. } + | SketchParams::CountSketch { depth, .. } + | SketchParams::CmsWithHeap { depth, .. } + | SketchParams::CountSketchWithHeap { depth, .. } => *depth, + other => panic!("{other:?}"), + }) + .collect() + }; + assert_eq!(depths(SketchAlgorithm::Cms), [5, 7, 10, 12]); + assert_eq!(depths(SketchAlgorithm::CountSketch), [47, 81, 117, 155]); + assert_eq!(depths(SketchAlgorithm::CmsWithHeap), [5, 5, 5, 5]); + assert_eq!(parameter_list(SketchAlgorithm::Cms).len(), 28); + assert_eq!(parameter_list(SketchAlgorithm::CmsWithHeap).len(), 280); + assert_eq!(parameter_list(SketchAlgorithm::UnivMon).len(), 40); + } } diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index e5d6f529b..14c595bb8 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -203,6 +203,49 @@ impl QueryRecurrence { _ => None, } } + + /// The grid every evaluation with this recurrence and `range_evaluation` + /// falls on, in milliseconds, anchored at the epoch (every evaluation + /// time is a multiple of it). The times' grid is the greatest common + /// divisor of a [`RepeatedDemand::FixedIntervalAt`]'s interval and phase + /// (every 20 s at phase 5 s is on a 5 s grid), of the scheduled times, or + /// a one-off query's time. A range evaluation of more than one step also + /// takes the greatest common divisor with its step, since each run + /// evaluates at every step up to its end. `None` when the times are not + /// known: a fixed interval without a declared phase (Prometheus staggers + /// rule groups by a hash, so it does not mean phase 0), an estimated + /// rate, a one-off query without a time, an unknown recurrence, or a + /// zero interval. Pass 2's chain widths and the evaluation grids of + /// Stage 3's contexts, which + /// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on) + /// reads, both come from it. + pub fn evaluation_grid_ms(&self, range_evaluation: Option) -> Option { + let times_grid_ms = match self { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + evaluation_phase, .. + }) => greatest_common_divisor(self.fixed_interval_ms()?, evaluation_phase.0), + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => times + .iter() + .fold(0, |grid_ms, time| greatest_common_divisor(grid_ms, time.0)), + QueryRecurrence::OneTime { + execute_at: Some(time), + .. + } => time.0, + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(_) | RepeatedDemand::EstimatedRate(_), + ) + | QueryRecurrence::OneTime { + execute_at: None, .. + } + | QueryRecurrence::Unknown => return None, + }; + Some(match range_evaluation { + Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { + greatest_common_divisor(times_grid_ms, range.step_ms) + } + _ => times_grid_ms, + }) + } } /// The most evaluation steps one range request may have, as Prometheus @@ -363,34 +406,6 @@ pub struct QueryWorkloadEntry { pub range_evaluation: Option, } -impl QueryWorkloadEntry { - /// The grid every evaluation of this query falls on, in milliseconds, - /// anchored at the epoch (every evaluation time is a multiple of it): - /// its fixed interval ([`QueryRecurrence::fixed_interval_ms`]), or with - /// a range evaluation of more than one step the greatest common divisor - /// of the interval and the step, since each run evaluates at every step - /// up to its end; with a declared evaluation phase - /// ([`RepeatedDemand::FixedIntervalAt`]) also the greatest common - /// divisor with that phase (every 20 s at phase 5 s is on a 5 s grid). - /// `None` when it does not repeat at a positive fixed interval. - pub fn evaluation_grid_ms(&self) -> Option { - let interval_ms = self.recurrence.fixed_interval_ms()?; - let step_ms = match self.range_evaluation { - Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { - greatest_common_divisor(interval_ms, range.step_ms) - } - _ => interval_ms, - }; - let phase_ms = match &self.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) => evaluation_phase.0, - _ => 0, - }; - Some(greatest_common_divisor(step_ms, phase_ms)) - } -} - impl From<&BatchEntry> for QueryWorkloadEntry { fn from(entry: &BatchEntry) -> Self { Self { @@ -633,24 +648,35 @@ mod tests { assert_eq!(range.evaluations_per_request(), 61); let entry = QueryWorkloadEntry::from(&hour); assert_eq!(entry.range_evaluation, Some(range)); - // Refreshes every 30 s and steps every 60 s: evaluations on a 30 s grid. - assert_eq!(entry.evaluation_grid_ms(), Some(30_000)); - assert_eq!( - QueryWorkloadEntry::from(&plain).evaluation_grid_ms(), - Some(30_000) - ); + // A fixed interval without a declared phase: the evaluation times + // are not known, so there is no grid. + let grid = |entry: &QueryWorkloadEntry| { + entry.recurrence.evaluation_grid_ms(entry.range_evaluation) + }; + assert_eq!(grid(&entry), None); + assert_eq!(grid(&QueryWorkloadEntry::from(&plain)), None); + let every = |interval_ms, phase_ms| { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(phase_ms), + }) + }; + // Refreshes every 30 s at phase 0 and steps every 60 s: evaluations + // on a 30 s grid. + let stepped = QueryWorkloadEntry { + recurrence: every(30_000, 0), + ..entry.clone() + }; + assert_eq!(grid(&stepped), Some(30_000)); // At a declared phase, also on that phase from the epoch: every // minute at phase 125 s is on a 5 s grid; at phase 120 s, on 60 s. let at_phase = |phase_ms| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase_ms), - }), + recurrence: every(60_000, phase_ms), range_evaluation: None, ..entry.clone() }; - assert_eq!(at_phase(125_000).evaluation_grid_ms(), Some(5_000)); - assert_eq!(at_phase(120_000).evaluation_grid_ms(), Some(60_000)); + assert_eq!(grid(&at_phase(125_000)), Some(5_000)); + assert_eq!(grid(&at_phase(120_000)), Some(60_000)); // A one-step range (span 0) is one evaluation per request: its // step does not narrow the grid (refreshing every 60 s at a 45 s // step would otherwise give a 15 s grid for nothing). @@ -659,12 +685,10 @@ mod tests { step_ms: 45_000, span_ms: 0, }), - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(60_000), - )), + recurrence: every(60_000, 0), ..entry.clone() }; - assert_eq!(one_step.evaluation_grid_ms(), Some(60_000)); + assert_eq!(grid(&one_step), Some(60_000)); let validate = |range: RangeEvaluation| { let mut workload = base_workload(); diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 5dec0c4a7..c70d13489 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -6,7 +6,9 @@ use std::time::Duration; use asap_types::ir::operator::arguments::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; +use asap_types::ir::{ + ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, TimeRangeKind, +}; use asap_types::summary_type::{SketchAlgorithm, SummaryUpdate}; const MINUTE: u64 = 60_000; @@ -85,3 +87,213 @@ fn names_before_the_tumbling_window_rename_still_read() { operator ); } + +/// A scan with a `host` label, then the chain, the plain build and the +/// time selections over it by host. +fn scan_with_host() -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("host", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap() +} + +fn node(operator: Operator) -> Rc { + let node = OperatorNode::new_shared(operator).unwrap(); + node.validate_structure().unwrap(); + node +} + +/// The `SummaryAgg` (`chain` false) or the 1-min chain (`chain` true) of +/// `summary_type` by host over `input`. +fn by_host(input: Rc, summary_type: FieldDataType, chain: bool) -> Operator { + let reduction = Reduction::by(vec![1]); + let update = SummaryUpdate::column(ColumnRef::SampleValue); + Operator::ASAP(if chain { + ASAPOp::SummaryTumblingWindowChain { + child: input, + summary_type, + input: update, + reduction, + filter: None, + tumbling_window_width_ms: MINUTE, + } + } else { + ASAPOp::SummaryAgg { + child: input, + summary_type, + input: update, + reduction, + filter: None, + } + }) +} + +/// `TimeRange(5m, TimeShift(1h, input))`. +fn five_minutes_an_hour_ago(input: Rc) -> Rc { + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: asap_types::ir::operator::arguments::TimeShift { + offset_ms: 60 * MINUTE as i64, + at: None, + }, + child: input, + })); + node(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Range, + child: shifted, + })) +} + +fn merge(children: Vec>, group_by: Option>) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by }) +} + +/// A chain's output is a state with a time index, one row per window and +/// key: (window_end, keys, state), unique by (window_end, keys), for a +/// keyed chain and a per-series one alike. `window_end` replaces the +/// input's `ts`. The plain build by host has no time index. +#[test] +fn a_chain_is_a_state_with_a_time_index() { + let names = |node: &Rc| -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() + }; + let by_host_chain = node(by_host(scan_with_host(), kll(), true)); + let per_series = node(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan_with_host(), + summary_type: kll(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: MINUTE, + })); + for chain in [&by_host_chain, &per_series] { + assert_eq!(chain.result_kind, OperatorResultKind::State); + assert_eq!(chain.schema.time_index, Some(0)); + assert_eq!(names(chain), ["window_end", "host", "state"]); + assert_eq!(chain.schema.unique_keys, [vec![0, 1]]); + } + let build = node(by_host(scan_with_host(), kll(), false)); + assert_eq!(build.schema.time_index, None); + let without_labels = chain(scan()); + assert_eq!(names(&without_labels), ["window_end", "state"]); + assert_eq!(without_labels.schema.unique_keys, [vec![0]]); +} + +/// `TimeRange` and `TimeShift` accept a time-indexed state, a chain or a +/// selection over one, and keep it a state with the chain's schema; over +/// a state without a time index (a plain build) they are rejected. +#[test] +fn time_range_and_time_shift_select_a_chains_windows() { + let windows = node(by_host(scan_with_host(), kll(), true)); + let selected = five_minutes_an_hour_ago(windows.clone()); + assert_eq!(selected.result_kind, OperatorResultKind::State); + assert_eq!(selected.schema, windows.schema); + let build = node(by_host(scan_with_host(), kll(), false)); + let over_build = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Range, + child: build, + })) + .unwrap(); + assert!(over_build.validate_structure().is_err()); +} + +/// A `TimeRange` over a chain selects a range of windows; an instant one +/// is rejected. +#[test] +fn an_instant_time_range_over_a_chain_is_rejected() { + let instant = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Instant, + child: node(by_host(scan_with_host(), kll(), true)), + })) + .unwrap(); + assert!(instant.validate_structure().is_err()); +} + +/// Merging the windows `TimeRange(5m, TimeShift(1h, chain(X)))` selects +/// gives the state of `SummaryAgg(TimeRange(5m, TimeShift(1h, X)))`: the +/// same schema and kind, and so the same schema read out, finalized +/// (an exact sum) and rolled up. +#[test] +fn merged_windows_are_the_build_over_the_selected_range() { + let sum = FieldDataType::exact(asap_types::summary_type::ExactKind::Sum); + for summary_type in [kll(), sum.clone()] { + let merged = node(merge( + vec![five_minutes_an_hour_ago(node(by_host( + scan_with_host(), + summary_type.clone(), + true, + )))], + None, + )); + let build = node(by_host( + five_minutes_an_hour_ago(scan_with_host()), + summary_type.clone(), + false, + )); + assert_eq!(merged.schema, build.schema, "{summary_type:?}"); + assert_eq!(merged.result_kind, build.result_kind); + let read = |state: &Rc| { + node(if summary_type == sum { + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { + child: state.clone(), + }) + } else { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state.clone(), + query: asap_types::summary_type::SketchStatistic::Quantile { q: 0.95 }, + }) + }) + }; + let (read_merged, read_build) = (read(&merged), read(&build)); + assert_eq!(read_merged.schema, read_build.schema, "{summary_type:?}"); + assert_eq!(read_merged.result_kind, read_build.result_kind); + // Rolled up to one state over every host. + let (rolled_merged, rolled_build) = ( + node(merge(vec![merged], Some(vec![]))), + node(merge(vec![build], Some(vec![]))), + ); + assert_eq!(rolled_merged.schema, rolled_build.schema); + let (read_merged, read_build) = (read(&rolled_merged), read(&rolled_build)); + assert_eq!(read_merged.schema, read_build.schema, "{summary_type:?}"); + } +} + +/// A merge over a time-indexed state reads one `TimeRange` over a chain: +/// not the whole chain, not a `TimeShift` without a range, and not beside +/// another input. +#[test] +fn a_merge_over_a_chain_reads_one_time_range() { + let windows = || node(by_host(scan_with_host(), kll(), true)); + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: asap_types::ir::operator::arguments::TimeShift { + offset_ms: MINUTE as i64, + at: None, + }, + child: windows(), + })); + for children in [ + vec![windows()], + vec![shifted], + vec![ + five_minutes_an_hour_ago(windows()), + five_minutes_an_hour_ago(windows()), + ], + ] { + assert!(OperatorNode::new_shared(merge(children, None)).is_err()); + } + assert!( + OperatorNode::new_shared(merge(vec![five_minutes_an_hour_ago(windows())], None)).is_ok() + ); +}