From b65c3964cb29c0a99c44b52a9f68960e5f4e1c92 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:18:23 +0000 Subject: [PATCH] refactor(types): properties and requirements from #1207 (#1207 PR 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LogicalProperties::new(expression, parameters): a sketch build (a chain included) needs θ from parameter_list(algorithm) of its FieldDataType::Sketch(algorithm); an exact operator has none. - parameter_list(algorithm): structures only. Count-Min lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, 81, 117 and 155 (its median-of-rows δ levels 1e-2 … 1e-5); heap variants each of those with the 1-2-5 heap sizes; UnivMon 40 entries; KLL, DDSketch, HLL, KMV one entry per ε level. A test keeps the lists equal to today's sizing, which is test-only. - PhysicalProperties::new rejects (IngestionTime, OneEvaluation); producer_satisfies_physical_requirements(producer, consumer_timing). - PhysicalCandidate::delivers_on(candidates, context): None in every context, OneEvaluation only in Evaluation contexts, a chain with AcrossEvaluations when its width divides the grid's period and phase (or the refresh window's width and phase), any other AcrossEvaluations result everywhere when its group does not depend on the evaluation time, else when its group's evaluation grid holds the context's times; Materialized(v) contexts stay todo!() for PR 7. - Chains: one schema (window_end at column 0, keys, state), TimeRange and TimeShift over a chain select windows (an instant TimeRange is rejected), a SummaryMerge over them finalizes and rolls up as the chain's build over that range (the two PR 0 todos). - summary_type_of_operator; QueryRecurrence::evaluation_grid_ms is None for a fixed interval without a phase. --- crates/accuracy/src/guarantee/cardinality.rs | 2 + crates/accuracy/src/guarantee/cms.rs | 1 + crates/accuracy/src/guarantee/count_sketch.rs | 2 + crates/accuracy/src/guarantee/ddsketch.rs | 1 + crates/accuracy/src/guarantee/hll.rs | 1 + crates/accuracy/src/guarantee/kll.rs | 2 + crates/accuracy/src/guarantee/mod.rs | 109 +++++- crates/accuracy/src/guarantee/univmon.rs | 10 + crates/accuracy/src/propagation.rs | 95 +++--- crates/accuracy/src/sizing.rs | 31 +- crates/accuracy/src/test_support.rs | 18 +- crates/types/src/ir/operator/asap.rs | 172 ++++++++-- crates/types/src/ir/operator/non_asap.rs | 25 ++ .../src/ir/operator/tumbling_window_chain.rs | 36 +- crates/types/src/stages/logical/candidate.rs | 171 +++++++++- crates/types/src/stages/logical/memo/mod.rs | 3 +- crates/types/src/stages/physical/candidate.rs | 90 ++++- .../src/stages/physical/memo_candidates.rs | 320 ++++++++++++++++++ .../types/src/stages/physical/properties.rs | 61 ++-- .../types/src/summary_type/parameter_lists.rs | 299 +++++++++++++++- crates/types/src/workload/query.rs | 112 +++--- crates/types/tests/tumbling_window_chain.rs | 214 +++++++++++- 22 files changed, 1532 insertions(+), 243 deletions(-) diff --git a/crates/accuracy/src/guarantee/cardinality.rs b/crates/accuracy/src/guarantee/cardinality.rs index a437da6de..5e03bd0b6 100644 --- a/crates/accuracy/src/guarantee/cardinality.rs +++ b/crates/accuracy/src/guarantee/cardinality.rs @@ -33,6 +33,7 @@ use super::*; /// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. const KMV_MIN_K: u32 = 2; /// The largest `k` sizing returns. +#[cfg(test)] const KMV_MAX_K: u32 = 1 << 26; pub(super) fn guarantee( @@ -81,6 +82,7 @@ pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { /// The smallest `k` whose KMV failure probability at relative error `eps` /// is at most `delta` ([`kmv_failure_probability`]), up to `2^26`; a /// degenerate `eps` or `delta` saturates to `2^26`. +#[cfg(test)] pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return KMV_MAX_K; diff --git a/crates/accuracy/src/guarantee/cms.rs b/crates/accuracy/src/guarantee/cms.rs index 6895ac6e1..ad48f58c2 100644 --- a/crates/accuracy/src/guarantee/cms.rs +++ b/crates/accuracy/src/guarantee/cms.rs @@ -25,6 +25,7 @@ pub(super) fn guarantee( } /// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. +#[cfg(test)] pub(crate) fn cms_width(eps: f64) -> u32 { saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) } diff --git a/crates/accuracy/src/guarantee/count_sketch.rs b/crates/accuracy/src/guarantee/count_sketch.rs index 4f72dfd99..52d6e4b7c 100644 --- a/crates/accuracy/src/guarantee/count_sketch.rs +++ b/crates/accuracy/src/guarantee/count_sketch.rs @@ -57,12 +57,14 @@ fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { } /// CountSketch `L2` point-query width: ε = sqrt(3/w). +#[cfg(test)] pub(crate) fn count_sketch_width(eps: f64) -> u32 { saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) } /// The smallest positive odd depth whose binomial median failure /// probability at `width` is at most `delta`, capped at 255. +#[cfg(test)] pub(crate) fn count_sketch_depth(width: u32, delta: f64) -> u32 { if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { return 255; diff --git a/crates/accuracy/src/guarantee/ddsketch.rs b/crates/accuracy/src/guarantee/ddsketch.rs index 9cde75477..5f1a7310c 100644 --- a/crates/accuracy/src/guarantee/ddsketch.rs +++ b/crates/accuracy/src/guarantee/ddsketch.rs @@ -20,6 +20,7 @@ pub(super) fn guarantee( )) } +#[cfg(test)] pub(super) fn size_params(epsilon: f64) -> SketchParams { SketchParams::DDSketch { alpha: epsilon } } diff --git a/crates/accuracy/src/guarantee/hll.rs b/crates/accuracy/src/guarantee/hll.rs index 7da74066d..ea463ff96 100644 --- a/crates/accuracy/src/guarantee/hll.rs +++ b/crates/accuracy/src/guarantee/hll.rs @@ -255,6 +255,7 @@ pub fn min_precision( /// bound up to `⌈1/ε⌉` (18 when none up to 18 does). Pass 1 then raises /// the precision by the same rule ([`min_precision`]) when the source's own /// bound or a smaller failure probability needs it. +#[cfg(test)] pub(crate) fn hll_precision(eps: f64, delta: f64) -> u8 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return 18; diff --git a/crates/accuracy/src/guarantee/kll.rs b/crates/accuracy/src/guarantee/kll.rs index bc6f0f010..c450d4573 100644 --- a/crates/accuracy/src/guarantee/kll.rs +++ b/crates/accuracy/src/guarantee/kll.rs @@ -52,6 +52,7 @@ pub(crate) fn kll_rank_error_99(k: u32) -> f64 { /// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. Capped at /// [`KLL_MAX_K`], the largest `k` `asap_sketchlib` builds; an `eps` that /// needs more gets that `k`, whose guarantee then misses the target. +#[cfg(test)] pub(crate) fn kll_k(eps: f64) -> u32 { saturating_ceil( (KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / KLL_RANK_ERROR_EXPONENT_99), @@ -133,6 +134,7 @@ fn normal_quantile(p: f64) -> f64 { /// `k` for rank error `eps` at failure probability `delta`: the empirical /// 99% calibration for `delta >= 0.01`; below, the calibration at /// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. +#[cfg(test)] pub(crate) fn size_params(eps: f64, delta: f64) -> SketchParams { if delta.is_nan() || delta <= 0.0 || delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { return SketchParams::Kll { k: kll_k(eps) }; diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 9769da79f..fed1e51d9 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -7,6 +7,7 @@ use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; #[cfg(test)] use crate::requirements::meets; +#[cfg(test)] use asap_types::ir::operator::AggIntent; use asap_types::summary_type::SummaryTypeCapability; @@ -87,7 +88,10 @@ pub fn theoretical_guarantee( } } -#[allow(dead_code)] // Read again for the predefined θ lists (`parameter_list`), in #1207 PR 1. +/// Today's sizing: the parameters of algorithm `kind` for error `eps` at failure +/// probability `delta`. Test-only: the predefined lists (`parameter_list`) +/// are this sizing over an error grid, which a test keeps equal. +#[cfg(test)] pub(crate) fn size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -192,6 +196,7 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. +#[cfg(test)] pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { u32::try_from(k) .unwrap_or(u32::MAX) @@ -211,6 +216,108 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { mod tests { use super::*; + /// Each algorithm's predefined list (`parameter_list`) is today's sizing + /// over the error grid ε = 0.1 … 0.001 at failure probability 0.01, + /// smallest first, without repeats; KMV's and UnivMon's distinct counts + /// stop at ε = 0.01. Count-Min lists each sized width with depths 5, 7, + /// 10 and 12, Count Sketch with 47, 81, 117 and 155, and a heap variant + /// each of those with + /// every heap size of the 1-2-5 series 10 … 10 000. UnivMon's list also + /// combines each L2 shape with each distinct-count shape (28 entries). + #[test] + fn parameter_list_is_the_sizing_over_the_error_grid() { + use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; + use asap_types::summary_type::parameter_list; + let full_grid = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; + let distinct_grid = [0.1, 0.05, 0.02, 0.01]; + let cms_depths = [5, 7, 10, 12]; + let count_sketch_depths = [47, 81, 117, 155]; + let heap_sizes = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; + let l2 = AggIntent::FrequencyL2 { col: None }; + let top_1 = AggIntent::TopK { k: 1 }; + let quantile = default_quantile(0.5); + let distinct = default_cardinality(); + for (algorithm, intents) in [ + (SketchAlgorithm::Kll, vec![&quantile]), + (SketchAlgorithm::DDSketch, vec![&quantile]), + (SketchAlgorithm::Hll, vec![&distinct]), + (SketchAlgorithm::Kmv, vec![&distinct]), + (SketchAlgorithm::Cms, vec![&distinct]), + (SketchAlgorithm::CountSketch, vec![&distinct]), + (SketchAlgorithm::CmsWithHeap, vec![&top_1]), + (SketchAlgorithm::CountSketchWithHeap, vec![&top_1]), + (SketchAlgorithm::UnivMon, vec![&l2, &distinct, &quantile]), + ] { + let mut sized: Vec = Vec::new(); + for intent in intents { + let grid: &[f64] = match (algorithm, intent) { + ( + SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon, + AggIntent::Cardinality { .. }, + ) => &distinct_grid, + _ => &full_grid, + }; + for &eps in grid { + let listed: Vec = match size_params(algorithm, intent, eps, 0.01) + { + SketchParams::Cms { width, .. } => cms_depths + .iter() + .map(|&depth| SketchParams::Cms { width, depth }) + .collect(), + SketchParams::CountSketch { width, .. } => count_sketch_depths + .iter() + .map(|&depth| SketchParams::CountSketch { width, depth }) + .collect(), + SketchParams::CmsWithHeap { width, .. } => cms_depths + .iter() + .flat_map(|&depth| { + heap_sizes + .iter() + .map(move |&heap_size| SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }) + }) + .collect(), + SketchParams::CountSketchWithHeap { width, .. } => count_sketch_depths + .iter() + .flat_map(|&depth| { + heap_sizes.iter().map(move |&heap_size| { + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } + }) + }) + .collect(), + other => vec![other], + }; + for params in listed { + if !sized.contains(¶ms) { + sized.push(params); + } + } + } + } + let list = parameter_list(algorithm); + let (listed_sized, combined): (Vec<_>, Vec<_>) = + list.iter().partition(|params| sized.contains(params)); + assert_eq!( + format!("{listed_sized:?}"), + format!("{:?}", sized.iter().collect::>()), + "{algorithm:?}" + ); + let expected_combined = if algorithm == SketchAlgorithm::UnivMon { + 28 + } else { + 0 + }; + assert_eq!(combined.len(), expected_combined, "{algorithm:?}"); + } + } + /// Classic HLL's precision grows until its failure probability over the /// source's distinct bound meets the target, up to 18. #[test] diff --git a/crates/accuracy/src/guarantee/univmon.rs b/crates/accuracy/src/guarantee/univmon.rs index 2bf1d8a56..c0545c20c 100644 --- a/crates/accuracy/src/guarantee/univmon.rs +++ b/crates/accuracy/src/guarantee/univmon.rs @@ -17,18 +17,22 @@ //! (OctoSketch aggregators fed promoted deltas) are not covered: the //! executor rejects them. use super::*; +#[cfg(test)] use asap_types::ir::operator::AggIntent; /// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and /// seed 19 picks the layers a key reaches. +#[cfg(test)] pub(crate) const MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; /// The most rows an L2 sketch is sized with: the deepest odd shape within /// the 20 rows the executor builds ([`hash_independent`]), so the row median /// is one row's estimate, and within the hash budget at the more than 20 /// columns an L2 guarantee needs (5 column bits: 19 · 6 = 114 ≤ 128). +#[cfg(test)] const MAX_L2_ROWS: u32 = 19; /// Each row of a layer reads its column and sign bits from one hash. +#[cfg(test)] const HASH_BITS: u32 = 128; /// The per-row failure probability the L2 guarantee fixes: a row misses by /// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it @@ -39,6 +43,7 @@ const L2_ROW_FAILURE: f64 = 0.1; const DISTINCT_VARIANCE: f64 = 1.35; /// The smallest heap the distinct-count variance bound holds for. const DISTINCT_MIN_HEAP: u32 = 8; +#[cfg(test)] const CARD_CAP: u32 = 1 << 26; const L2_CONTRACT: &str = "univmon_layer0_f2_median_chebyshev_binomial_v1"; @@ -50,6 +55,7 @@ const MAX_DISTINCT: &str = "max_distinct"; /// The shape baseline for readouts without a size model: the total (exact /// whatever the shape) and entropy (uncertified). +#[cfg(test)] const BASELINE: SketchParams = SketchParams::UnivMon { heap_size: 256, sketch_rows: 5, @@ -147,6 +153,7 @@ fn l2_failure_probability(rows: u32) -> f64 { /// clamped to the hash budget: with more rows than [`MAX_L2_ROWS`], or more /// columns than `rows` leave bits for, the shape is the largest one built /// and its guarantee misses the target. +#[cfg(test)] fn l2_shape(eps: f64, delta: f64) -> (u32, u32) { let rows = (1..=MAX_L2_ROWS) .find(|&rows| l2_failure_probability(rows) <= delta) @@ -186,6 +193,7 @@ fn incomplete_top_layer_bound(heap_size: u32) -> f64 { /// The Chebyshev share of the distinct count's failure probability at /// heap `k` and relative error `eps`: `1.35/(k ε²)`. +#[cfg(test)] fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { DISTINCT_VARIANCE / (f64::from(heap_size) * eps * eps) } @@ -194,6 +202,7 @@ fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { /// `eps · F0` with probability at most `delta`, top-layer completeness /// included: `1.35/(k ε²) + (e/4)^(k/2) ≤ δ`. Capped; a capped heap's /// guarantee misses the target. +#[cfg(test)] fn distinct_heap(eps: f64, delta: f64) -> u32 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return CARD_CAP; @@ -320,6 +329,7 @@ pub(crate) fn distinct_guarantee( /// counter; /// - total and entropy: the baseline shape (the total is exact; entropy /// has no certified bound). +#[cfg(test)] pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { match intent { AggIntent::FrequencyL2 { .. } => { diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index c50268744..41fd07d4a 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -11,7 +11,6 @@ use asap_types::accuracy::{ ProbabilityExpr, }; use asap_types::ir::operator::arguments::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::merged_chain; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; @@ -528,32 +527,31 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> } /// The guarantee of an estimate or finalized value read from merged -/// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range (the whole chain's, or -/// its newest tumbling windows') when the -/// family's merged error for that read is exact or the same as that build's -/// (the family-capability table); none when it is approximate (evidence +/// tumbling windows (a state whose [`OperatorNode::summary_build`] is a +/// chain): the guarantee of the same read of one build over the time range +/// the merged windows cover when the +/// summary type's merged error for that read is exact or the same as that build's +/// (the summary-type capability table); none when it is approximate (evidence /// only, for plan selection to judge) or the family has no merge. `None` for /// any node that does not read merged tumbling windows. fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let merged = match node.asap()? { + let (state, query) = match node.asap()? { ASAPOp::SummaryEstimate { summary_input, query, - } => { - let chain = merged_chain(summary_input)?; - let family = chain.asap()?.produced_state()?; - SummaryTypeCapability::of(family).merged_error(query) - } - ASAPOp::FinalizeExactAccumulator { child } => { - let chain = merged_chain(child)?; - let family = chain.asap()?.produced_state()?; - // An exact accumulator's merged state is exact. - let merged = SummaryTypeCapability::of(family).merge_inputs; - merged.map(|_| MergedError::Exact) - } + } => (summary_input, Some(query)), + ASAPOp::FinalizeExactAccumulator { child } => (child, None), _ => return None, }; + let ASAPOp::SummaryTumblingWindowChain { summary_type, .. } = state.summary_build()? else { + return None; + }; + let capability = SummaryTypeCapability::of(&summary_type); + let merged = match query { + Some(query) => capability.merged_error(query), + // An exact accumulator's merged state is exact. + None => capability.merge_inputs.map(|_| MergedError::Exact), + }; if !matches!( merged, Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) @@ -663,40 +661,43 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match merged_chain(summary_input) - .unwrap_or_else(|| Rc::clone(summary_input)) - .asap()? - { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. + }) if (0.0..=1.0).contains(q) => { + let build = match summary_input.summary_build()? { + chain @ ASAPOp::SummaryTumblingWindowChain { .. } => chain, + _ => summary_input.asap()?.clone(), + }; + match &build { + ASAPOp::SummaryAgg { + input: + SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + .. + }, + filter: None, + child, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + input: + SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + .. + }, + filter: None, + child, + .. + } => estimates_raw_data_samples_of(child), + _ => None, } - | ASAPOp::SummaryTumblingWindowChain { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - }, + } Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { match summary_input.asap()? { ASAPOp::SummaryAgg { diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index 25e5c2410..ddb42ffea 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -1,9 +1,10 @@ -//! Sizing: the statistic a summary is read for, and the default sketch -//! parameters for an `(ε, δ)`. +//! The statistic a summary is read for. (Today's sizing for an `(ε, δ)` is +//! test-only, `estimators::size_params`: the predefined lists are checked +//! against it.) use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::ColumnRef; -use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::summary_type::SketchStatistic; /// The statistic a summary is read for to answer `intent`, or `None` when /// no sketch statistic serves the intent. @@ -22,47 +23,31 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { }) } -/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved -/// `(eps, delta)` accuracy budget. -/// -/// Each formula inverts the sketch family's standard error bound to the -/// smallest parameter satisfying the target, clamped to the family's sane -/// range. A non-positive ε saturates to the clamp maximum (tightest -/// allowed). -#[allow(dead_code)] // Read by `test_support` until #1207 PR 1 replaces it with `parameter_list`. -pub(crate) fn default_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - crate::guarantee::size_params(kind, intent, eps, delta) -} - #[cfg(test)] mod tests { use super::*; + use asap_types::summary_type::{SketchAlgorithm, SketchParams}; /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. #[test] fn epsilon_delta_sizes_cms_depth() { let intent = AggIntent::Count; assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), + crate::guarantee::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), SketchParams::Cms { width: 2719, depth: 7 } // ⌈e/0.001⌉, ⌈ln 1000⌉ ); assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), + crate::guarantee::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), SketchParams::Cms { width: 2719, depth: 5 } ); assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), + crate::guarantee::size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), SketchParams::Cms { width: 272, depth: 5 diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs index 67de5f443..bb1ba7986 100644 --- a/crates/accuracy/src/test_support.rs +++ b/crates/accuracy/src/test_support.rs @@ -6,10 +6,9 @@ use std::rc::Rc; use asap_types::accuracy::AccuracyGuarantee; use asap_types::ir::operator::arguments::Reduction; -use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{OperatorNode, ScalarExpr}; -use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate}; +use asap_types::summary_type::{SketchStatistic, SummaryUpdate}; use crate::composition::StreamNorm; use crate::propagation; @@ -70,11 +69,6 @@ pub fn source_distinct_guarantee( propagation::source_distinct_guarantee(family, query, input) } -/// `crate::guarantee::cms::cms_width`. -pub fn cms_width(eps: f64) -> u32 { - crate::guarantee::cms::cms_width(eps) -} - /// `crate::guarantee::cms::cms_depth`. pub fn cms_depth(delta: f64) -> u32 { crate::guarantee::cms::cms_depth(delta) @@ -85,16 +79,6 @@ pub fn kll_rank_error_99(k: u32) -> f64 { crate::guarantee::kll::kll_rank_error_99(k) } -/// `crate::sizing::default_size_params`. -pub fn default_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - crate::sizing::default_size_params(kind, intent, eps, delta) -} - /// `crate::composition::QUOTIENT_OPERAND_DOMAINS`. pub const QUOTIENT_OPERAND_DOMAINS: &str = crate::composition::QUOTIENT_OPERAND_DOMAINS; diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 9b2dc957f..fc7f44900 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -8,6 +8,8 @@ use serde::{Deserialize, Serialize}; use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::non_asap::NonASAPOp; +use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::arguments::Reduction; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; @@ -76,8 +78,9 @@ pub enum ASAPOp> { /// `reduction` and `filter`, built over each window /// `(k·w, (k+1)·w]` of `child`, `w` = `tumbling_window_width_ms`, on a /// grid anchored at the epoch. Each window is built when it ends. Output: - /// one row per window, (window end, keys, state), a state with a time - /// index. It has no range and no window count: a `TimeRange` (and + /// one row per window and key, (window_end, keys, state), a state with + /// a time index (`window_end`, column 0), unique by (window_end, keys). + /// It has no range and no window count: a `TimeRange` (and /// `TimeShift`) over it selects the windows a query reads, and how many /// windows it retains is $h_v$, a Stage 3 choice. // Written as `PaneChain` before #1163; the aliases read those documents. @@ -282,13 +285,13 @@ impl ASAPOp { summary_type: family, reduction, .. - } - | SummaryTumblingWindowChain { + } => summary_agg_schema(child.schema(), family, reduction)?, + SummaryTumblingWindowChain { child, summary_type: family, reduction, .. - } => summary_agg_schema(child.schema(), family, reduction)?, + } => chain_schema(child.schema(), family, reduction)?, SummaryEstimate { summary_input, query, @@ -337,9 +340,8 @@ impl ASAPOp { if let Some((build, schema)) = rolled_up_build(child) { return finalized_schema(Some(&build), &schema); } - if super::tumbling_window_chain::merged_chain(child).is_some() { - todo!("#1207 PR 1: finalize the windows a TimeRange over a chain selects as one build over that TimeRange") - } + // The windows a `TimeRange` over a chain selects, merged, + // finalize as the chain's build over that range. finalized_schema(summary_build(child).as_ref(), child.schema())? } MaintainPopulation { child, .. } => child.schema().clone(), @@ -411,9 +413,22 @@ impl ASAPOp { } => { self.validate_inputs()?; let input = children[0].schema(); - match group_by { - None => input.clone(), - Some(keys) => regrouped_schema(input, keys)?, + match (group_by, summary_build(&children[0])) { + (Some(keys), _) => regrouped_schema(input, keys)?, + // The windows a `TimeRange` over a chain selects merge + // into the state of the chain's build over that range, + // which has no window column (a `TimeRange` and a + // `TimeShift` keep their input's schema). + ( + None, + Some(SummaryTumblingWindowChain { + child, + summary_type, + reduction, + .. + }), + ) => summary_agg_schema(child.schema(), &summary_type, &reduction)?, + (None, _) => input.clone(), } } SummarySubtract { .. } @@ -475,6 +490,19 @@ impl ASAPOp { "summary merge requires exactly one state column".into(), )); } + // Over a time-indexed state: the windows one `TimeRange` + // over a chain selects, not the whole chain. + if children.iter().any(is_time_indexed_state) + && !matches!( + children.as_slice(), + [child] if matches!(child.operator(), Operator::NonASAP(NonASAPOp::TimeRange { .. })) + ) + { + return Err(SchemaDerivationError::InvalidScalarSignature( + "a summary merge over a time-indexed state reads one TimeRange over a chain" + .into(), + )); + } for child in children { needs_state(child, "SummaryMerge")?; if child.schema() != first.schema() { @@ -656,22 +684,31 @@ fn rolled_up_build( return None; }; let input = selected_from(input).unwrap_or_else(|| input.clone()); - let build = match input.operator() { - Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => { + // A build, or the windows a `TimeRange` over a chain selects, merged: + // the chain's build over that range, whose schema is the build's over + // the chain's input. + let build = match (input.operator(), summary_build(&input)) { + (Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }), _) + | (_, Some(op @ ASAPOp::SummaryTumblingWindowChain { .. })) => { op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) } - _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { - todo!("#1207 PR 1: roll up the windows a TimeRange over a chain selects as one build over that TimeRange") - } _ => return None, }; - let ASAPOp::SummaryAgg { + let (ASAPOp::SummaryAgg { + child, + summary_type, + input: update, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { child, - summary_type: family, + summary_type, input: update, reduction, filter, - } = build + .. + }) = build else { return None; }; @@ -684,11 +721,11 @@ fn rolled_up_build( }) .collect::>>()?; let reduction = Reduction::by(mapped); - let schema = summary_agg_schema(child.schema(), &family, &reduction).ok()?; + let schema = summary_agg_schema(child.schema(), &summary_type, &reduction).ok()?; Some(( ASAPOp::SummaryAgg { child, - summary_type: family, + summary_type, input: update, reduction, filter, @@ -723,6 +760,39 @@ pub(crate) fn summary_agg_schema( Ok(schema) } +/// The schema of a `SummaryTumblingWindowChain` of `summary_type` by `reduction` +/// over rows of schema `child`: a state with a time index, one row per +/// window and key, (window_end, keys…, state). `window_end` (column 0, the +/// time index) is computed, so it takes no name or table from `child`; the +/// build's own time column, which a per-entity build keeps, is dropped. The +/// unique key is (window_end, keys). +fn chain_schema( + child: &Schema, + summary_type: &FieldDataType, + reduction: &Reduction, +) -> Result { + let mut schema = summary_agg_schema(child, summary_type, reduction)?; + let time = child + .time_index + .and_then(|index| child.fields.get(index)) + .ok_or_else(|| { + SchemaDerivationError::InvalidScalarSignature( + "a tumbling-window chain reads rows with a time column".into(), + ) + })?; + if let Some(build_time) = schema.time_index.take() { + schema.fields.remove(build_time); + } + schema + .fields + .insert(0, Field::new("window_end", time.dtype.clone(), false)); + schema.time_index = Some(0); + schema.unique_keys = vec![(0..schema.fields.len()) + .filter(|&column| schema.fields[column].is_plain()) + .collect()]; + Ok(schema) +} + /// The schema `FinalizeExactAccumulator` derives over state of /// `state_schema` that `build` (the state's `SummaryAgg`, if any) builds: /// each exact aggregate replaced by the value it finalizes to. @@ -730,14 +800,24 @@ fn finalized_schema( build: Option<&ASAPOp>, state_schema: &Schema, ) -> Result { - let value_result = if let Some(ASAPOp::SummaryAgg { - child: source, - summary_type: FieldDataType::ExactAggregate(kind, _), - input, - reduction, - filter, - .. - }) = build + let value_result = if let Some( + ASAPOp::SummaryAgg { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + }, + ) = build { { use crate::ir::operator::AggIntent; @@ -843,22 +923,24 @@ fn finalized_data_type(kind: &crate::summary_type::ExactKind) -> DataType { } } -/// The states `node` selects from when it is a `Filter` that keeps whole -/// summary states by their plain columns (its output is state); `None` for +/// The states `node` selects from when it keeps whole summary states (its +/// output is state): a `Filter` by their plain columns, or a `TimeRange` or +/// `TimeShift` over a time-indexed state, which selects windows. `None` for /// any other input. pub fn selected_from(node: &I) -> Option { match node.operator() { - Operator::NonASAP(crate::ir::NonASAPOp::Filter { child, .. }) - if node.result_kind() == OperatorResultKind::State => - { - Some(child) - } + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. }, + ) if node.result_kind() == OperatorResultKind::State => Some(child), _ => None, } } /// The build whose state this summary state carries: a `SummaryAgg` or -/// `SummaryTumblingWindowChain` itself, or the first input's build under +/// `SummaryTumblingWindowChain` itself, the chain under the `TimeRange` and +/// `TimeShift`s that select its windows, or the first input's build under /// `SummaryMerge`s (merged inputs have one definition, which summary /// coverage checks). `None` under a regrouping merge, whose states no build /// carries (its coverage's definition is the build it stands for), and for @@ -873,6 +955,22 @@ pub fn summary_build(state: &I) -> Option> { group_by: None, .. }) => summary_build(children.first()?), + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => summary_build(&child), + _ => None, + } +} + +/// The summary type a build operator builds: a `SummaryAgg`'s or a +/// `SummaryTumblingWindowChain`'s `summary_type`; `None` for every other +/// operator. +pub fn summary_type_of_operator(operator: &Operator) -> Option<&FieldDataType> { + match operator { + Operator::ASAP( + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, + ) => Some(summary_type), _ => None, } } diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 7eb6bbfba..2312fd73c 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use super::input::OperatorInput; use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::agg_intent::AggIntent; use crate::ir::operator::arguments::{ BinaryOpKind, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, @@ -921,6 +922,12 @@ impl NonASAPOp { TimeRange { child, .. } if child.result_kind() == OperatorResultKind::Relation => { OperatorResultKind::Relation } + // Over a time-indexed state (a chain): the windows it selects. + TimeRange { child, .. } | TimeShift { child, .. } + if child.result_kind() == OperatorResultKind::State => + { + OperatorResultKind::State + } TimeRange { kind, .. } => match kind { TimeRangeKind::Instant => OperatorResultKind::InstantVector, TimeRangeKind::Range => OperatorResultKind::RangeVector, @@ -1162,6 +1169,24 @@ impl NonASAPOp { return Err(invalid("label expression requires a string")); } } + TimeRange { child, .. } | TimeShift { child, .. } + if child.result_kind() == OperatorResultKind::State => + { + if !is_time_indexed_state(child) { + return Err(invalid( + "a time selection over state requires a time-indexed state", + )); + } + if matches!( + self, + TimeRange { + kind: TimeRangeKind::Instant, + .. + } + ) { + return Err(invalid("a time selection over a chain is a range")); + } + } TimeRange { child, .. } => { if child.result_kind() != OperatorResultKind::Relation { instant(child)?; diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index aca6285b0..065a3a310 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -1,33 +1,23 @@ //! Tumbling-window chains ([`ASAPOp::SummaryTumblingWindowChain`]): one //! row per window of width `w`, on a grid anchored at the epoch, a state //! with a time index. A chain has no range and no window count: a -//! `TimeRange` (and `TimeShift`) over it selects the windows a query reads -//! (#1207 PR 1), and how many windows it retains is $h_v$ (#1207 PR 7). +//! `TimeRange` (and `TimeShift`s) over it select the windows a query reads, +//! a `SummaryMerge` over that selection is the state of the chain's build +//! over the same range, and how many windows it retains is $h_v$ (#1207 +//! PR 7). use super::asap::ASAPOp; use super::input::OperatorInput; use super::node::Operator; +use super::non_asap::NonASAPOp; -/// The `SummaryTumblingWindowChain` a summary state carries merged: the -/// chain itself, or the one chain a `SummaryMerge` that keeps its groups -/// reads. `None` for any other input, a rollup (regrouping merge) included. -pub fn merged_chain(state: &I) -> Option { +/// Whether `state` is a state with a time index: a chain, or a `TimeRange` +/// or `TimeShift` over one. +pub(crate) fn is_time_indexed_state(state: &I) -> bool { match state.operator() { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), - Operator::ASAP(ASAPOp::SummaryMerge { - children, - group_by: None, - .. - }) => match children.as_slice() { - [child] - if matches!( - child.operator(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) => - { - Some(child.clone()) - } - _ => None, - }, - _ => None, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => true, + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => is_time_indexed_state(&child), + _ => false, } } diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs index 61d59a103..618c228bc 100644 --- a/crates/types/src/stages/logical/candidate.rs +++ b/crates/types/src/stages/logical/candidate.rs @@ -2,10 +2,12 @@ //! (concepts doc `memo-groups-properties-requirements.md`, section 3; //! #1203). +use crate::ir::operator::asap::summary_type_of_operator; +use crate::ir::schema::FieldDataType; use crate::ir::Operator; use crate::ir::SchemaDerivationError; use crate::stages::{LogicalRequirements, MemoGroupId}; -use crate::summary_type::SketchParams; +use crate::summary_type::{parameter_list, SketchParams}; /// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). @@ -90,8 +92,21 @@ impl LogicalProperties { expression: &CandidateExpression, parameters: Option, ) -> Result { - let _ = (expression, parameters); - todo!("#1207 PR 1: reject invalid parameters") + let algorithm = match summary_type_of_operator(&expression.operator) { + Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), + _ => None, + }; + match (algorithm, ¶meters) { + (Some(_), None) => return Err(LogicalPropertiesError::SketchBuildWithoutParameters), + (None, Some(_)) => return Err(LogicalPropertiesError::ParametersOnExactOperator), + (Some(algorithm), Some(parameters)) => { + if !parameter_list(algorithm).contains(parameters) { + return Err(LogicalPropertiesError::ParametersNotInList); + } + } + (None, None) => {} + } + Ok(Self { parameters }) } /// $\boldsymbol\theta_v$. @@ -111,3 +126,153 @@ pub enum LogicalPropertiesError { #[error("the parameters are not in their algorithm's predefined list")] ParametersNotInList, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::ASAPOp; + use crate::summary_type::SketchAlgorithm; + + use crate::ir::operator::arguments::Reduction; + + use crate::ir::scalar::expr_ir::ColumnRef; + + use crate::ir::schema::FieldDataType; + use crate::summary_type::{ExactKind, HydraParams, SummaryUpdate}; + + /// `SummaryAgg` over group 0 building a state of `summary_type`. + fn summary_agg(summary_type: FieldDataType) -> CandidateExpression { + CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryAgg { + child: MemoGroupId(0), + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + } + } + + const KLL_269: SketchParams = SketchParams::Kll { k: 269 }; + + const CMS: SketchParams = SketchParams::Cms { + width: 272, + depth: 5, + }; + + const HYDRA_CMS: SketchParams = SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 1_024, + }); + + /// `LogicalProperties::new` over every operator kind and parameters: a + /// sketch build (a chain included) needs parameters of its algorithm from + /// its list; parameters are rejected on an operator that builds no + /// sketch. + #[test] + fn logical_properties_new_over_every_operator_and_parameters() { + use LogicalPropertiesError as Invalid; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let kll_chain = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }), + }; + let readout = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: MemoGroupId(0), + query: crate::summary_type::SketchStatistic::Quantile { q: 0.5 }, + }), + }; + // k = 270 is KLL's structure but not in its list. + let parameters = [ + None, + Some(KLL_269), + Some(CMS), + Some(HYDRA_CMS), + Some(SketchParams::Kll { k: 270 }), + ]; + let no_parameters = Err(Invalid::ParametersOnExactOperator); + let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); + let not_in_list = Err(Invalid::ParametersNotInList); + let exact = [ + Ok(()), + no_parameters, + no_parameters, + no_parameters, + no_parameters, + ]; + // Another algorithm's parameters are not in KLL's list either. + let kll_build = [ + needs_parameters, + Ok(()), + not_in_list, + not_in_list, + not_in_list, + ]; + // Per operator: the result for each of `parameters`. + let table = [ + (summary_agg(kll), kll_build), + // No Hydra θ is listed yet, so no Hydra build is valid. + ( + summary_agg(FieldDataType::Sketch(SketchAlgorithm::HydraCms)), + [ + needs_parameters, + not_in_list, + not_in_list, + not_in_list, + not_in_list, + ], + ), + (summary_agg(FieldDataType::exact(ExactKind::Sum)), exact), + (kll_chain, kll_build), + (readout, exact), + ]; + for (expression, by_parameters) in table { + for (parameters, expected) in parameters.iter().zip(by_parameters) { + let properties = LogicalProperties::new(&expression, parameters.clone()); + assert_eq!( + properties.as_ref().map(|_| ()).map_err(|e| *e), + expected, + "{:?} {parameters:?}", + expression.operator.kind_name() + ); + if let Ok(properties) = properties { + assert_eq!(properties.parameters(), parameters.as_ref()); + } + } + } + } + + /// `summary_type_of_operator` reads the summary type of a `SummaryAgg` + /// and of a chain, and of no other operator. + #[test] + fn summary_type_of_operator_reads_the_builds_summary_type() { + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![MemoGroupId(0)], + group_by: None, + }); + assert_eq!( + summary_type_of_operator(&summary_agg(kll.clone()).operator), + Some(&kll) + ); + assert_eq!(summary_type_of_operator(&chain), Some(&kll)); + assert_eq!(summary_type_of_operator(&merge), None); + } +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index efef26be1..cdd7e212d 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -82,8 +82,7 @@ impl Memo { /// The candidate `id`. pub fn candidate(&self, id: CandidateForMemoGroupId) -> &CandidateForMemoGroup { - let _ = id; - todo!("#1207 PR 2") + &self.groups[id.group.0 as usize].candidates[id.index_in_group] } /// Replaces candidate `id`'s input requirements $r^{\log}_{c,i}$, one diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index 130817db2..c5891c112 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -2,8 +2,9 @@ //! valid $p^{\mathrm{phys}} = (\tau, \ell)$ (#1200). use crate::ir::properties::ExecutionTiming; -use crate::stages::PhysicalProperties; +use crate::ir::{ASAPOp, Operator}; use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; +use crate::stages::{Lifetime, PhysicalProperties}; /// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -20,24 +21,70 @@ pub struct PhysicalCandidate { impl PhysicalCandidate { /// Whether a consumer running in `context` can read this producer (the - /// concepts doc's alignment table, section 2). True for $\ell$ = None, - /// and for `OneEvaluation` in the same context. A chain producer - /// (`SummaryTumblingWindowChain`) with $\ell$ = `AcrossEvaluations`, with - /// $w$ the chain operator's width, needs every time of the context on - /// its grid: $w$ divides the context's period and phase. Any other - /// producer with $\ell$ = `AcrossEvaluations` is stored on its group's - /// evaluation grid (`evaluation_grid_of_group`): it delivers on every - /// context when the group does not depend on the evaluation time, else - /// when that grid holds every time of the context. In - /// `Materialized(v)`, the context's times are v's, read from - /// `candidates`. + /// concepts doc's alignment table, section 2): true for $\ell$ = None, + /// which runs in the consumer's own context; for `OneEvaluation` only in + /// an `Evaluation` context, the one evaluation its result belongs to. + /// With $\ell$ = `AcrossEvaluations`, every execution time of the + /// context must be a time the producer has stored: for a chain + /// (`SummaryTumblingWindowChain`) of width $w$, $w$ divides an + /// evaluation grid's period and phase, or a refresh window's width and + /// phase; any other result is stored on its group's evaluation grid + /// (`evaluation_grid_of_group`), which must hold every time of the + /// context, unless the group does not depend on the evaluation time + /// (then it delivers on every context). pub fn delivers_on( &self, candidates: &PhysicalASAPMemoCandidates, context: &ExecutionContext, ) -> bool { - let _ = (candidates, context); - todo!("#1207 PR 1: the alignment table of section 2") + match self.physical_properties.lifetime() { + Lifetime::None => true, + Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), + Lifetime::AcrossEvaluations(_) => { + let logical = &candidates.logical_candidates; + let group = self.logical_candidate_id.group; + let chain_width_ms = match logical + .memo + .candidate(self.logical_candidate_id) + .expression + .operator + { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(tumbling_window_width_ms), + _ => None, + }; + let depends_on_evaluation_time = logical.memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time; + if chain_width_ms.is_none() && !depends_on_evaluation_time { + return true; + } + let (period_ms, phase_ms) = match context { + ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), + ExecutionContext::IngestionPipeline(window) => { + (window.width_ms, window.phase_ms) + } + ExecutionContext::Materialized(_) => { + todo!( + "#1207 PR 7: the times of a Materialized(v) context, v's own schedule" + ) + } + }; + match chain_width_ms { + Some(width_ms) => { + period_ms.is_multiple_of(width_ms) && phase_ms.is_multiple_of(width_ms) + } + None => { + logical.evaluation_grid_of_group[group.0 as usize].is_some_and(|grid| { + period_ms.is_multiple_of(grid.period_ms) + && phase_ms % grid.period_ms == grid.phase_ms % grid.period_ms + }) + } + } + } + } } } @@ -48,6 +95,17 @@ pub fn producer_satisfies_physical_requirements( producer: &PhysicalCandidate, consumer_timing: ExecutionTiming, ) -> bool { - let _ = (producer, consumer_timing); - todo!("#1207 PR 1: the timing x lifetime table of section 2") + use ExecutionTiming::{IngestionTime, QueryTime}; + match ( + producer.physical_properties.timing(), + producer.physical_properties.lifetime(), + ) { + (QueryTime, _) => consumer_timing == QueryTime, + (IngestionTime, Lifetime::None) => consumer_timing == IngestionTime, + // Materialized at ingestion: read at ingestion time and at query time. + (IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by `PhysicalProperties::new`") + } + } } diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index f3b1ec719..0aaf77cd4 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -100,3 +100,323 @@ pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, } references } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::properties::ExecutionTiming; + use crate::ir::{ASAPOp, Operator}; + use crate::stages::{ + producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, + Lifetime, PhysicalProperties, + }; + + use crate::plan::materialized_view::RefreshWindow; + + use crate::ir::operator::arguments::{Reduction, Source}; + + use crate::ir::scalar::expr_ir::ColumnRef; + + use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; + use crate::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; + + use crate::ir::{NonASAPOp, TimeRangeKind}; + + use crate::stages::StorageMedium; + + use crate::stages::physical::execution_context::EvaluationGrid; + + use crate::stages::{ + CandidateExpression, CandidateForMemoGroup, LogicalASAPMemoCandidates, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, + }; + + use ExecutionTiming::{IngestionTime, QueryTime}; + + /// The KLL build over the scan: not a chain. + const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(1), + index_in_group: 0, + }; + + /// The chain of tumbling windows 5 s wide over the scan. + const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(2), + index_in_group: 0, + }; + + /// The chain of tumbling windows 10 s wide over the scan. + const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(3), + index_in_group: 0, + }; + + /// X = the scan's last 5 minutes (`m[5m]`): a result that depends on + /// the evaluation time and is not a chain. + const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(4), + index_in_group: 0, + }; + + /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains + /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last + /// 5 minutes, each with its own expression as its one candidate. Built + /// through `find_or_add_group` and `add_candidate` (#1207 PR 2). + fn memo() -> Memo { + let mut memo = Memo::default(); + let scan = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let chain = |tumbling_window_width_ms| { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms, + }) + }; + for operator in [build, chain(5_000), chain(10_000)] { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new( + &expression, + Some(SketchParams::Kll { k: 269 }), + ) + .unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(group, candidate).unwrap(); + } + let operator = Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(group, candidate).unwrap(); + memo + } + + /// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; + /// `delivers_on` reads nothing else. + fn candidates( + memo: Memo, + evaluation_grid_of_group: Vec>, + ) -> PhysicalASAPMemoCandidates { + PhysicalASAPMemoCandidates { + logical_candidates: LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group, + }, + physical_candidates: vec![], + pruned_candidates: vec![], + edges: vec![], + admissible_edges_into_input: HashMap::new(), + admissible_edges_out_of_candidate: vec![], + } + } + + fn candidate( + logical_candidate_id: CandidateForMemoGroupId, + timing: ExecutionTiming, + lifetime: Lifetime, + ) -> PhysicalCandidate { + PhysicalCandidate { + logical_candidate_id, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + } + } + + fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms, + }, + } + } + + fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) + } + + const MEMORY: StorageMedium = StorageMedium::Memory; + + /// A `None` producer runs in the consumer's context and delivers on + /// every one; a `OneEvaluation` producer only on an evaluation context, + /// the one evaluation its result belongs to. Neither reads the memo. + #[test] + fn delivers_on_by_lifetime_without_windows() { + let candidates = candidates(Memo::default(), vec![]); + let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); + let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); + let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); + for (producer, context, delivers) in [ + (&unmaterialized, evaluation(7_000, 3_000), true), + (&unmaterialized, ingestion(5_000, 0), true), + (&unmaterialized, materialized, true), + (&one_evaluation, evaluation(7_000, 3_000), true), + (&one_evaluation, ingestion(5_000, 0), false), + (&one_evaluation, materialized, false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} in {context:?}", + producer.physical_properties + ); + } + } + + /// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase + /// 0) delivers when $w$ divides the context's period (or refresh-window + /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s + /// serves both. + #[test] + #[ignore = "filled in by #1207 PR 2 (#1230)"] + fn delivers_on_a_chain_when_its_width_divides_the_context_times() { + let candidates = candidates(memo(), vec![None; 5]); + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let ingestion_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for (producer, context, delivers) in [ + // Evaluation grids: w divides period and phase. + (&across_5s, evaluation(10_000, 0), true), + (&across_5s, evaluation(10_000, 5_000), true), + (&across_5s, evaluation(10_000, 2_000), false), + (&across_5s, evaluation(7_000, 0), false), + (&ingestion_10s, evaluation(10_000, 0), true), + (&ingestion_10s, evaluation(10_000, 5_000), false), + // Refresh windows: w divides width and phase. + (&across_5s, ingestion(5_000, 0), true), + (&across_5s, ingestion(10_000, 5_000), true), + (&across_5s, ingestion(10_000, 2_500), false), + (&ingestion_10s, ingestion(5_000, 0), false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} {:?} in {context:?}", + producer.logical_candidate_id, + producer.physical_properties + ); + } + } + + /// A stored result that is not a chain delivers when its group's + /// evaluation grid holds every time of the context: X = + /// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is + /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or + /// another phase. Without a known grid it serves none. + #[test] + #[ignore = "filled in by #1207 PR 2 (#1230)"] + fn delivers_on_a_stored_result_on_its_group_grid() { + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); + let without_grid = candidates(memo(), vec![None; 5]); + let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + for (candidates, context, delivers) in [ + (&on_grid, evaluation(60_000, 0), true), + (&on_grid, evaluation(120_000, 0), true), + (&on_grid, evaluation(120_000, 60_000), true), + (&on_grid, evaluation(30_000, 0), false), + (&on_grid, evaluation(60_000, 15_000), false), + (&on_grid, ingestion(60_000, 0), true), + (&without_grid, evaluation(60_000, 0), false), + ] { + assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + } + } + + /// `producer_satisfies_physical_requirements` over every valid producer state and every valid + /// consumer state: a query-time producer serves only query-time + /// consumers, an ingestion-time producer that is not materialized only + /// ingestion-time consumers, and one materialized across evaluations + /// serves both. Chain widths do not matter: alignment is Stage 3's + /// per-context check. + #[test] + fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { + let states = |medium| { + [ + candidate(BUILD, QueryTime, Lifetime::None), + candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), + candidate(BUILD, IngestionTime, Lifetime::None), + candidate( + WINDOWS_5S, + IngestionTime, + Lifetime::AcrossEvaluations(medium), + ), + ] + }; + // Rows: producer state; columns: consumer state, in `states` order. + let expected = [ + [true, true, true, false, false], + [true, true, true, false, false], + [true, true, true, false, false], + [false, false, false, true, true], + [true, true, true, true, true], + ]; + for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (producer, row) in states(producer_medium).iter().zip(expected) { + for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { + assert_eq!( + producer_satisfies_physical_requirements( + producer, + consumer.physical_properties.timing() + ), + satisfies, + "{:?} -> {:?}", + producer.physical_properties, + consumer.physical_properties + ); + } + } + } + } + } +} diff --git a/crates/types/src/stages/physical/properties.rs b/crates/types/src/stages/physical/properties.rs index df1869ed4..3cd8f2a40 100644 --- a/crates/types/src/stages/physical/properties.rs +++ b/crates/types/src/stages/physical/properties.rs @@ -64,8 +64,12 @@ impl PhysicalProperties { timing: ExecutionTiming, lifetime: Lifetime, ) -> Result { - let _ = (timing, lifetime); - todo!("#1207 PR 1: reject the invalid combination of section 2") + match (timing, lifetime) { + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + } + _ => Ok(Self { timing, lifetime }), + } } /// $\tau_v$. @@ -89,32 +93,35 @@ mod tests { use super::*; use ExecutionTiming::{IngestionTime, QueryTime}; - const MEMORY: StorageMedium = StorageMedium::Memory; - - /// The one invalid combination of the concepts doc, section 2. - #[test] - fn new_rejects_one_evaluation_at_ingestion() { - assert_eq!( - PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), - Err(PhysicalPropertiesError::OneEvaluationAtIngestion) - ); - } - - /// The valid (τ, ℓ) pairs of section 2. + /// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for + /// both media, and keeps what it was given. #[test] - fn new_accepts_each_valid_pair() { - for (timing, lifetime) in [ - (QueryTime, Lifetime::None), - (IngestionTime, Lifetime::None), - (QueryTime, Lifetime::OneEvaluation(MEMORY)), - (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), - ( - IngestionTime, - Lifetime::AcrossEvaluations(StorageMedium::Disk), - ), - ] { - let properties = PhysicalProperties::new(timing, lifetime).unwrap(); - assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + fn new_over_every_timing_and_lifetime() { + for medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (timing, lifetime, valid) in [ + (QueryTime, Lifetime::None, true), + (QueryTime, Lifetime::OneEvaluation(medium), true), + (QueryTime, Lifetime::AcrossEvaluations(medium), true), + (IngestionTime, Lifetime::None, true), + (IngestionTime, Lifetime::OneEvaluation(medium), false), + (IngestionTime, Lifetime::AcrossEvaluations(medium), true), + ] { + let properties = PhysicalProperties::new(timing, lifetime); + match valid { + true => { + let properties = properties.unwrap(); + assert_eq!( + (properties.timing(), properties.lifetime()), + (timing, lifetime) + ); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + false => assert_eq!( + properties, + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ), + } + } } } } diff --git a/crates/types/src/summary_type/parameter_lists.rs b/crates/types/src/summary_type/parameter_lists.rs index c3ec86d4c..f87db5dbe 100644 --- a/crates/types/src/summary_type/parameter_lists.rs +++ b/crates/types/src/summary_type/parameter_lists.rs @@ -6,7 +6,302 @@ use super::{SketchAlgorithm, SketchParams}; /// The algorithm's predefined $\boldsymbol\theta$ list: structures only; the /// error of each, for a statistic at a failure probability, is its guarantee /// (`asap_accuracy::theoretical_guarantee`). +/// +/// The widths, `k`s, precisions, `alpha`s and heap sizes are today's sizing +/// (`asap_accuracy`'s test-only `size_params`) over the errors ε = 0.1, +/// 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, without +/// repeats; an `asap_accuracy` test keeps them equal. Distinct counts (KMV, +/// and UnivMon's distinct-count shapes) stop at ε = 0.01. Where δ is set by +/// a structural parameter, the list covers the δ levels with it: Count-Min +/// lists each width with each depth of [`CMS_DEPTHS`], Count Sketch with +/// each of [`COUNT_SKETCH_DEPTHS`]. A +/// UnivMon's list is its L2-norm shapes, its distinct-count shapes, each L2 +/// shape combined with each distinct-count shape, then its shape for every +/// other statistic. A heap variant combines each width × depth with each +/// heap size of [`HEAP_SIZES`]: a top-k needs heap_size ≥ k, and the next +/// listed step up is at most 2.5× too large. Empty for an algorithm without +/// an error model (the Hydra algorithms included), so no build of it is a +/// valid candidate. pub fn parameter_list(algorithm: SketchAlgorithm) -> &'static [SketchParams] { - let _ = algorithm; - todo!("#1207 PR 1: the predefined θ list per algorithm") + match algorithm { + SketchAlgorithm::Kll => &KLL_PARAMETERS, + SketchAlgorithm::DDSketch => &DDSKETCH_PARAMETERS, + SketchAlgorithm::Hll => &HLL_PARAMETERS, + SketchAlgorithm::Kmv => &KMV_PARAMETERS, + SketchAlgorithm::Cms => &CMS_PARAMETERS, + SketchAlgorithm::CountSketch => &COUNT_SKETCH_PARAMETERS, + SketchAlgorithm::CmsWithHeap => &CMS_WITH_HEAP_PARAMETERS, + SketchAlgorithm::CountSketchWithHeap => &COUNT_SKETCH_WITH_HEAP_PARAMETERS, + SketchAlgorithm::UnivMon => &UNIVMON_PARAMETERS, + // Listed, not `_`: an algorithm `asap_sketchlib` adds fails to + // compile here until it gets a list or is listed without one. + SketchAlgorithm::CountSketchWithL2Norm + | SketchAlgorithm::CmsWithSpaceSaving + | SketchAlgorithm::CountSketchWithSpaceSaving + | SketchAlgorithm::CmsFold + | SketchAlgorithm::CountSketchFold + | SketchAlgorithm::HllHip + | SketchAlgorithm::KllDynamic + | SketchAlgorithm::SpaceSaving + | SketchAlgorithm::Elastic + | SketchAlgorithm::Coco + | SketchAlgorithm::Bloom + | SketchAlgorithm::SetAggregator + | SketchAlgorithm::UnivMonTwoTier + | SketchAlgorithm::UnivMonQ + | SketchAlgorithm::HydraCms + | SketchAlgorithm::HydraCountSketch + | SketchAlgorithm::HydraKll + | SketchAlgorithm::HydraHll + | SketchAlgorithm::HydraUnivMon => &[], + } +} + +const KLL_PARAMETERS: [SketchParams; 7] = [ + SketchParams::Kll { k: 26 }, + SketchParams::Kll { k: 52 }, + SketchParams::Kll { k: 132 }, + SketchParams::Kll { k: 269 }, + SketchParams::Kll { k: 547 }, + SketchParams::Kll { k: 1_404 }, + SketchParams::Kll { k: 2_863 }, +]; + +const DDSKETCH_PARAMETERS: [SketchParams; 7] = [ + SketchParams::DDSketch { alpha: 0.1 }, + SketchParams::DDSketch { alpha: 0.05 }, + SketchParams::DDSketch { alpha: 0.02 }, + SketchParams::DDSketch { alpha: 0.01 }, + SketchParams::DDSketch { alpha: 0.005 }, + SketchParams::DDSketch { alpha: 0.002 }, + SketchParams::DDSketch { alpha: 0.001 }, +]; + +/// HLL's precision saturates at 18 from ε = 0.01 on. +const HLL_PARAMETERS: [SketchParams; 4] = [ + SketchParams::Hll { precision: 13 }, + SketchParams::Hll { precision: 15 }, + SketchParams::Hll { precision: 17 }, + SketchParams::Hll { precision: 18 }, +]; + +const KMV_PARAMETERS: [SketchParams; 4] = [ + SketchParams::Kmv { k: 1_118 }, + SketchParams::Kmv { k: 4_323 }, + SketchParams::Kmv { k: 26_656 }, + SketchParams::Kmv { k: 106_263 }, +]; + +/// The depths each Count-Min width is listed with: δ = e^−d ≤ 1e-2, 1e-3, +/// 1e-4 and 1e-5. +const CMS_DEPTHS: [u32; 4] = [5, 7, 10, 12]; +/// The depths each Count Sketch width is listed with, for the same δ +/// levels: Count Sketch reads the median of its rows, each wrong with +/// probability about 1/3, so δ = P(Binomial(d, 1/3) ≥ (d+1)/2), the +/// smallest odd d at or below each level. +const COUNT_SKETCH_DEPTHS: [u32; 4] = [47, 81, 117, 155]; + +const CMS_WIDTHS: [u32; 7] = [28, 55, 136, 272, 544, 1_360, 2_719]; +const COUNT_SKETCH_WIDTHS: [u32; 7] = [300, 1_200, 7_500, 30_000, 120_000, 750_000, 3_000_000]; + +const CMS_PARAMETERS: [SketchParams; 28] = with_depths( + &CMS_WIDTHS, + &CMS_DEPTHS, + SketchParams::Cms { width: 0, depth: 0 }, +); +const COUNT_SKETCH_PARAMETERS: [SketchParams; 28] = with_depths( + &COUNT_SKETCH_WIDTHS, + &COUNT_SKETCH_DEPTHS, + SketchParams::CountSketch { width: 0, depth: 0 }, +); + +/// Each of `widths` with each of `depths`, by width, then depth, as +/// `shape`'s variant (`Cms` or `CountSketch`). +const fn with_depths( + widths: &[u32; 7], + depths: &[u32; 4], + shape: SketchParams, +) -> [SketchParams; 28] { + const UNSET: SketchParams = SketchParams::Cms { width: 0, depth: 0 }; + let mut list = [UNSET; 28]; + let mut width = 0; + while width < widths.len() { + let mut depth = 0; + while depth < depths.len() { + let (width_value, depth_value) = (widths[width], depths[depth]); + list[width * depths.len() + depth] = match shape { + SketchParams::Cms { .. } => SketchParams::Cms { + width: width_value, + depth: depth_value, + }, + SketchParams::CountSketch { .. } => SketchParams::CountSketch { + width: width_value, + depth: depth_value, + }, + _ => panic!("a Count-Min or Count Sketch shape"), + }; + depth += 1; + } + width += 1; + } + list +} + +/// The heap sizes of a heap variant's list: the 1-2-5 series from 10 to +/// 10 000, so a top-k with k ≤ 10 000 has a listed heap. +const HEAP_SIZES: [u32; 10] = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; + +const CMS_WITH_HEAP_PARAMETERS: [SketchParams; 280] = with_heap_sizes(&CMS_PARAMETERS); +const COUNT_SKETCH_WITH_HEAP_PARAMETERS: [SketchParams; 280] = + with_heap_sizes(&COUNT_SKETCH_PARAMETERS); + +/// Each `Cms` (or `CountSketch`) shape of `shapes` with each heap size of +/// [`HEAP_SIZES`], as `CmsWithHeap` (or `CountSketchWithHeap`). +const fn with_heap_sizes(shapes: &[SketchParams; 28]) -> [SketchParams; 280] { + const UNSET: SketchParams = SketchParams::Cms { width: 0, depth: 0 }; + let mut list = [UNSET; 280]; + let mut shape = 0; + while shape < shapes.len() { + let mut heap = 0; + while heap < HEAP_SIZES.len() { + let heap_size = HEAP_SIZES[heap]; + list[shape * HEAP_SIZES.len() + heap] = match shapes[shape] { + SketchParams::Cms { width, depth } => SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + SketchParams::CountSketch { width, depth } => SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + }, + _ => panic!("a heap variant extends a Cms or CountSketch shape"), + }; + heap += 1; + } + shape += 1; + } + list +} + +/// UnivMon's L2-norm column counts and distinct-count heap sizes. +const UNIVMON_L2_COLUMNS: [u32; 7] = [624, 2_367, 14_349, 56_817, 226_130, 1_409_067, 5_630_630]; +const UNIVMON_DISTINCT_HEAP_SIZES: [u32; 4] = [13_500, 54_000, 337_500, 1_350_000]; + +const UNIVMON_PARAMETERS: [SketchParams; 40] = univmon_parameters(); + +/// The L2-norm shapes, the distinct-count shapes, then each L2 column +/// count × distinct heap size (by column count, then heap size), last the +/// shape for every other statistic. +const fn univmon_parameters() -> [SketchParams; 40] { + const fn univmon( + heap_size: u32, + sketch_rows: u32, + sketch_cols: u32, + layers: u8, + ) -> SketchParams { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + } + } + const BASELINE: SketchParams = univmon(256, 5, 1_024, 16); + let mut list = [BASELINE; 40]; + let mut next = 0; + let mut columns = 0; + while columns < UNIVMON_L2_COLUMNS.len() { + list[next] = univmon(1, 5, UNIVMON_L2_COLUMNS[columns], 1); + next += 1; + columns += 1; + } + let mut heap = 0; + while heap < UNIVMON_DISTINCT_HEAP_SIZES.len() { + list[next] = univmon(UNIVMON_DISTINCT_HEAP_SIZES[heap], 1, 1, 19); + next += 1; + heap += 1; + } + columns = 0; + while columns < UNIVMON_L2_COLUMNS.len() { + heap = 0; + while heap < UNIVMON_DISTINCT_HEAP_SIZES.len() { + list[next] = univmon( + UNIVMON_DISTINCT_HEAP_SIZES[heap], + 5, + UNIVMON_L2_COLUMNS[columns], + 19, + ); + next += 1; + heap += 1; + } + columns += 1; + } + list +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each predefined list holds one parameter type, without repeats, and + /// is empty exactly for the algorithms without an error model. Count-Min + /// lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, + /// 81, 117 and 155, and a + /// heap variant each of those with every heap size; UnivMon's list has 40 + /// entries. + #[test] + fn parameter_list_holds_its_algorithm_without_repeats() { + let listed = [ + SketchAlgorithm::Kll, + SketchAlgorithm::DDSketch, + SketchAlgorithm::Hll, + SketchAlgorithm::Kmv, + SketchAlgorithm::Cms, + SketchAlgorithm::CountSketch, + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + SketchAlgorithm::UnivMon, + ]; + for algorithm in listed.iter().copied().chain([ + SketchAlgorithm::HydraCms, + SketchAlgorithm::HydraCountSketch, + SketchAlgorithm::HydraKll, + SketchAlgorithm::HllHip, + ]) { + let list = parameter_list(algorithm); + assert_eq!( + list.is_empty(), + !listed.contains(&algorithm), + "{algorithm:?}" + ); + for (i, parameters) in list.iter().enumerate() { + assert_eq!( + std::mem::discriminant(parameters), + std::mem::discriminant(&list[0]) + ); + assert!(!list[..i].contains(parameters), "{parameters:?} repeats"); + } + } + let depths = |algorithm| -> Vec { + parameter_list(algorithm) + .iter() + .take(4) + .map(|parameters| match parameters { + SketchParams::Cms { depth, .. } + | SketchParams::CountSketch { depth, .. } + | SketchParams::CmsWithHeap { depth, .. } + | SketchParams::CountSketchWithHeap { depth, .. } => *depth, + other => panic!("{other:?}"), + }) + .collect() + }; + assert_eq!(depths(SketchAlgorithm::Cms), [5, 7, 10, 12]); + assert_eq!(depths(SketchAlgorithm::CountSketch), [47, 81, 117, 155]); + assert_eq!(depths(SketchAlgorithm::CmsWithHeap), [5, 5, 5, 5]); + assert_eq!(parameter_list(SketchAlgorithm::Cms).len(), 28); + assert_eq!(parameter_list(SketchAlgorithm::CmsWithHeap).len(), 280); + assert_eq!(parameter_list(SketchAlgorithm::UnivMon).len(), 40); + } } diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index e5d6f529b..14c595bb8 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -203,6 +203,49 @@ impl QueryRecurrence { _ => None, } } + + /// The grid every evaluation with this recurrence and `range_evaluation` + /// falls on, in milliseconds, anchored at the epoch (every evaluation + /// time is a multiple of it). The times' grid is the greatest common + /// divisor of a [`RepeatedDemand::FixedIntervalAt`]'s interval and phase + /// (every 20 s at phase 5 s is on a 5 s grid), of the scheduled times, or + /// a one-off query's time. A range evaluation of more than one step also + /// takes the greatest common divisor with its step, since each run + /// evaluates at every step up to its end. `None` when the times are not + /// known: a fixed interval without a declared phase (Prometheus staggers + /// rule groups by a hash, so it does not mean phase 0), an estimated + /// rate, a one-off query without a time, an unknown recurrence, or a + /// zero interval. Pass 2's chain widths and the evaluation grids of + /// Stage 3's contexts, which + /// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on) + /// reads, both come from it. + pub fn evaluation_grid_ms(&self, range_evaluation: Option) -> Option { + let times_grid_ms = match self { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + evaluation_phase, .. + }) => greatest_common_divisor(self.fixed_interval_ms()?, evaluation_phase.0), + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => times + .iter() + .fold(0, |grid_ms, time| greatest_common_divisor(grid_ms, time.0)), + QueryRecurrence::OneTime { + execute_at: Some(time), + .. + } => time.0, + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(_) | RepeatedDemand::EstimatedRate(_), + ) + | QueryRecurrence::OneTime { + execute_at: None, .. + } + | QueryRecurrence::Unknown => return None, + }; + Some(match range_evaluation { + Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { + greatest_common_divisor(times_grid_ms, range.step_ms) + } + _ => times_grid_ms, + }) + } } /// The most evaluation steps one range request may have, as Prometheus @@ -363,34 +406,6 @@ pub struct QueryWorkloadEntry { pub range_evaluation: Option, } -impl QueryWorkloadEntry { - /// The grid every evaluation of this query falls on, in milliseconds, - /// anchored at the epoch (every evaluation time is a multiple of it): - /// its fixed interval ([`QueryRecurrence::fixed_interval_ms`]), or with - /// a range evaluation of more than one step the greatest common divisor - /// of the interval and the step, since each run evaluates at every step - /// up to its end; with a declared evaluation phase - /// ([`RepeatedDemand::FixedIntervalAt`]) also the greatest common - /// divisor with that phase (every 20 s at phase 5 s is on a 5 s grid). - /// `None` when it does not repeat at a positive fixed interval. - pub fn evaluation_grid_ms(&self) -> Option { - let interval_ms = self.recurrence.fixed_interval_ms()?; - let step_ms = match self.range_evaluation { - Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { - greatest_common_divisor(interval_ms, range.step_ms) - } - _ => interval_ms, - }; - let phase_ms = match &self.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) => evaluation_phase.0, - _ => 0, - }; - Some(greatest_common_divisor(step_ms, phase_ms)) - } -} - impl From<&BatchEntry> for QueryWorkloadEntry { fn from(entry: &BatchEntry) -> Self { Self { @@ -633,24 +648,35 @@ mod tests { assert_eq!(range.evaluations_per_request(), 61); let entry = QueryWorkloadEntry::from(&hour); assert_eq!(entry.range_evaluation, Some(range)); - // Refreshes every 30 s and steps every 60 s: evaluations on a 30 s grid. - assert_eq!(entry.evaluation_grid_ms(), Some(30_000)); - assert_eq!( - QueryWorkloadEntry::from(&plain).evaluation_grid_ms(), - Some(30_000) - ); + // A fixed interval without a declared phase: the evaluation times + // are not known, so there is no grid. + let grid = |entry: &QueryWorkloadEntry| { + entry.recurrence.evaluation_grid_ms(entry.range_evaluation) + }; + assert_eq!(grid(&entry), None); + assert_eq!(grid(&QueryWorkloadEntry::from(&plain)), None); + let every = |interval_ms, phase_ms| { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(phase_ms), + }) + }; + // Refreshes every 30 s at phase 0 and steps every 60 s: evaluations + // on a 30 s grid. + let stepped = QueryWorkloadEntry { + recurrence: every(30_000, 0), + ..entry.clone() + }; + assert_eq!(grid(&stepped), Some(30_000)); // At a declared phase, also on that phase from the epoch: every // minute at phase 125 s is on a 5 s grid; at phase 120 s, on 60 s. let at_phase = |phase_ms| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase_ms), - }), + recurrence: every(60_000, phase_ms), range_evaluation: None, ..entry.clone() }; - assert_eq!(at_phase(125_000).evaluation_grid_ms(), Some(5_000)); - assert_eq!(at_phase(120_000).evaluation_grid_ms(), Some(60_000)); + assert_eq!(grid(&at_phase(125_000)), Some(5_000)); + assert_eq!(grid(&at_phase(120_000)), Some(60_000)); // A one-step range (span 0) is one evaluation per request: its // step does not narrow the grid (refreshing every 60 s at a 45 s // step would otherwise give a 15 s grid for nothing). @@ -659,12 +685,10 @@ mod tests { step_ms: 45_000, span_ms: 0, }), - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(60_000), - )), + recurrence: every(60_000, 0), ..entry.clone() }; - assert_eq!(one_step.evaluation_grid_ms(), Some(60_000)); + assert_eq!(grid(&one_step), Some(60_000)); let validate = |range: RangeEvaluation| { let mut workload = base_workload(); diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 5dec0c4a7..c70d13489 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -6,7 +6,9 @@ use std::time::Duration; use asap_types::ir::operator::arguments::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; +use asap_types::ir::{ + ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, TimeRangeKind, +}; use asap_types::summary_type::{SketchAlgorithm, SummaryUpdate}; const MINUTE: u64 = 60_000; @@ -85,3 +87,213 @@ fn names_before_the_tumbling_window_rename_still_read() { operator ); } + +/// A scan with a `host` label, then the chain, the plain build and the +/// time selections over it by host. +fn scan_with_host() -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("host", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap() +} + +fn node(operator: Operator) -> Rc { + let node = OperatorNode::new_shared(operator).unwrap(); + node.validate_structure().unwrap(); + node +} + +/// The `SummaryAgg` (`chain` false) or the 1-min chain (`chain` true) of +/// `summary_type` by host over `input`. +fn by_host(input: Rc, summary_type: FieldDataType, chain: bool) -> Operator { + let reduction = Reduction::by(vec![1]); + let update = SummaryUpdate::column(ColumnRef::SampleValue); + Operator::ASAP(if chain { + ASAPOp::SummaryTumblingWindowChain { + child: input, + summary_type, + input: update, + reduction, + filter: None, + tumbling_window_width_ms: MINUTE, + } + } else { + ASAPOp::SummaryAgg { + child: input, + summary_type, + input: update, + reduction, + filter: None, + } + }) +} + +/// `TimeRange(5m, TimeShift(1h, input))`. +fn five_minutes_an_hour_ago(input: Rc) -> Rc { + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: asap_types::ir::operator::arguments::TimeShift { + offset_ms: 60 * MINUTE as i64, + at: None, + }, + child: input, + })); + node(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Range, + child: shifted, + })) +} + +fn merge(children: Vec>, group_by: Option>) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by }) +} + +/// A chain's output is a state with a time index, one row per window and +/// key: (window_end, keys, state), unique by (window_end, keys), for a +/// keyed chain and a per-series one alike. `window_end` replaces the +/// input's `ts`. The plain build by host has no time index. +#[test] +fn a_chain_is_a_state_with_a_time_index() { + let names = |node: &Rc| -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() + }; + let by_host_chain = node(by_host(scan_with_host(), kll(), true)); + let per_series = node(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan_with_host(), + summary_type: kll(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: MINUTE, + })); + for chain in [&by_host_chain, &per_series] { + assert_eq!(chain.result_kind, OperatorResultKind::State); + assert_eq!(chain.schema.time_index, Some(0)); + assert_eq!(names(chain), ["window_end", "host", "state"]); + assert_eq!(chain.schema.unique_keys, [vec![0, 1]]); + } + let build = node(by_host(scan_with_host(), kll(), false)); + assert_eq!(build.schema.time_index, None); + let without_labels = chain(scan()); + assert_eq!(names(&without_labels), ["window_end", "state"]); + assert_eq!(without_labels.schema.unique_keys, [vec![0]]); +} + +/// `TimeRange` and `TimeShift` accept a time-indexed state, a chain or a +/// selection over one, and keep it a state with the chain's schema; over +/// a state without a time index (a plain build) they are rejected. +#[test] +fn time_range_and_time_shift_select_a_chains_windows() { + let windows = node(by_host(scan_with_host(), kll(), true)); + let selected = five_minutes_an_hour_ago(windows.clone()); + assert_eq!(selected.result_kind, OperatorResultKind::State); + assert_eq!(selected.schema, windows.schema); + let build = node(by_host(scan_with_host(), kll(), false)); + let over_build = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Range, + child: build, + })) + .unwrap(); + assert!(over_build.validate_structure().is_err()); +} + +/// A `TimeRange` over a chain selects a range of windows; an instant one +/// is rejected. +#[test] +fn an_instant_time_range_over_a_chain_is_rejected() { + let instant = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Instant, + child: node(by_host(scan_with_host(), kll(), true)), + })) + .unwrap(); + assert!(instant.validate_structure().is_err()); +} + +/// Merging the windows `TimeRange(5m, TimeShift(1h, chain(X)))` selects +/// gives the state of `SummaryAgg(TimeRange(5m, TimeShift(1h, X)))`: the +/// same schema and kind, and so the same schema read out, finalized +/// (an exact sum) and rolled up. +#[test] +fn merged_windows_are_the_build_over_the_selected_range() { + let sum = FieldDataType::exact(asap_types::summary_type::ExactKind::Sum); + for summary_type in [kll(), sum.clone()] { + let merged = node(merge( + vec![five_minutes_an_hour_ago(node(by_host( + scan_with_host(), + summary_type.clone(), + true, + )))], + None, + )); + let build = node(by_host( + five_minutes_an_hour_ago(scan_with_host()), + summary_type.clone(), + false, + )); + assert_eq!(merged.schema, build.schema, "{summary_type:?}"); + assert_eq!(merged.result_kind, build.result_kind); + let read = |state: &Rc| { + node(if summary_type == sum { + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { + child: state.clone(), + }) + } else { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state.clone(), + query: asap_types::summary_type::SketchStatistic::Quantile { q: 0.95 }, + }) + }) + }; + let (read_merged, read_build) = (read(&merged), read(&build)); + assert_eq!(read_merged.schema, read_build.schema, "{summary_type:?}"); + assert_eq!(read_merged.result_kind, read_build.result_kind); + // Rolled up to one state over every host. + let (rolled_merged, rolled_build) = ( + node(merge(vec![merged], Some(vec![]))), + node(merge(vec![build], Some(vec![]))), + ); + assert_eq!(rolled_merged.schema, rolled_build.schema); + let (read_merged, read_build) = (read(&rolled_merged), read(&rolled_build)); + assert_eq!(read_merged.schema, read_build.schema, "{summary_type:?}"); + } +} + +/// A merge over a time-indexed state reads one `TimeRange` over a chain: +/// not the whole chain, not a `TimeShift` without a range, and not beside +/// another input. +#[test] +fn a_merge_over_a_chain_reads_one_time_range() { + let windows = || node(by_host(scan_with_host(), kll(), true)); + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: asap_types::ir::operator::arguments::TimeShift { + offset_ms: MINUTE as i64, + at: None, + }, + child: windows(), + })); + for children in [ + vec![windows()], + vec![shifted], + vec![ + five_minutes_an_hour_ago(windows()), + five_minutes_an_hour_ago(windows()), + ], + ] { + assert!(OperatorNode::new_shared(merge(children, None)).is_err()); + } + assert!( + OperatorNode::new_shared(merge(vec![five_minutes_an_hour_ago(windows())], None)).is_ok() + ); +}