diff --git a/Cargo.lock b/Cargo.lock index b49997d52..84b535f2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -466,7 +466,6 @@ name = "asap-stage2-physical-optimization" version = "0.1.0" dependencies = [ "asap-types", - "thiserror 2.0.18", ] [[package]] diff --git a/crates/cost/src/node/state_bytes.rs b/crates/cost/src/node/state_bytes.rs deleted file mode 100644 index 7c3d6f667..000000000 --- a/crates/cost/src/node/state_bytes.rs +++ /dev/null @@ -1,675 +0,0 @@ -//! The bytes one summary or exact state holds and its update operations -//! per input row, from its parameters and the rows it folds. - -use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::{DataType, FieldDataType, Schema}; -use asap_types::summary_type::KLL_MAX_K; -use asap_types::summary_type::{SketchParams, EXACT_ACCUMULATOR_STATE_BYTES}; - -/// Plain values are 8 bytes, strings 16; summary columns are sized apart. -pub fn row_bytes(schema: &Schema) -> u64 { - schema - .fields - .iter() - .map(|f| match &f.dtype { - FieldDataType::Plain(DataType::Utf8) => 16, - _ => 8, - }) - .sum::() - .max(1) -} - -/// Exact state that keeps every value or item of its group, so grows with -/// the data; other exact accumulators are fixed-size. -pub fn keeps_every_value(measure: &AggIntent) -> bool { - matches!( - measure, - AggIntent::Quantile { .. } - | AggIntent::TopK { .. } - | AggIntent::Cardinality { .. } - | AggIntent::FrequencyL2 { .. } - | AggIntent::FrequencyEntropy { .. } - ) -} - -/// Update operations per input row and bytes per state, from the sketch's -/// parameters and `input_rows` (the rows one state folds). The bytes are -/// `asap_sketchlib`'s `estimated_memory_bytes` for the type the executor -/// builds (Count-Min and Count Sketch over `f64` counters), given what the -/// state is assumed to hold: -/// - a Count-Min / Count Sketch top-k heap full of keys of -/// `ASSUMED_HEAP_KEY_BYTES`; each UnivMon layer's heap full of keys of -/// `ASSUMED_VALUE_KEY_BYTES`; -/// - DDSketch and KMV: the rows the state folds, at least one (any number -/// when unknown), each a distinct item for KMV; a DDSketch's values span -/// `DDSKETCH_ASSUMED_VALUE_RANGE` and fill its negative store too -/// unless proven non-negative. -/// -/// KLL is priced without the library's estimate: its fixed bytes -/// (`kll_fixed_bytes`: the struct and its level array) plus the 8-byte -/// item array as `asap_sketchlib` grows it for the rows the state folds, at -/// most its full capacity (`kll_buffer_items`). -/// -/// An exact accumulator's state is the executor's per-group state, -/// [`EXACT_ACCUMULATOR_STATE_BYTES`] for every kind. -/// -/// Update operations per input row: one counter per row of depth for -/// Count-Min and Count-Sketch, plus one heap update for their top-k variants; -/// for UnivMon, the same per layer it updates (`univmon_expected_layers`); -/// 1 for every other sketch. -pub fn summary_shape(family: &FieldDataType, input_rows: StateInputRows) -> (f64, u64) { - match family { - FieldDataType::Sketch(..) => { - let _ = input_rows; - todo!("#1207 PR 6: the state shape of θ from the candidate's LogicalProperties::parameters") - } - FieldDataType::ExactAggregate(..) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), - _ => (1.0, 8), - } -} - -/// What one summary state's bytes depend on besides its parameters (only -/// a KLL's, a DDSketch's and a KMV's do): the input rows it folds and the -/// sign of their values. -/// The rows are whatever its producer emits (raw data samples, rates, sums), -/// not a summary sample. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct StateInputRows { - /// Input rows one state folds, the average over its states rounded up; - /// `None` when unknown, which sizes it for every value it can be assumed - /// to see. - pub rows_per_state: Option, - /// Whether every value folded is proven non-negative (raw data samples of - /// sources declared non-negative), so a DDSketch fills no negative store. - pub non_negative: bool, -} - -impl StateInputRows { - /// Unknown rows of unknown sign: the most a state is assumed to hold. - pub const UNKNOWN: Self = Self { - rows_per_state: None, - non_negative: false, - }; -} - -/// [`summary_shape`] of one sketch: its update operations per row and its -/// bytes per state. With [`StateInputRows::UNKNOWN`], the bytes are also the -/// size the measured table interpolates between -/// (`crate::measured`). -pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, u64) { - use asap_sketchlib::{ - Classic, CountMinF64, CountMinSketchWithHeap, CountSketch, CountSketchWithHeap, DataInput, - DdSketch, HyperLogLog, Kmv, RegularPath, UnivMon, Vector2D, - }; - let heap_key = DataInput::Bytes(&[0; ASSUMED_HEAP_KEY_BYTES]); - let rows = input_rows.rows_per_state.map_or(usize::MAX, |rows| { - usize::try_from(rows.max(1)).unwrap_or(usize::MAX) - }); - match params { - SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), - SketchParams::Cms { width, depth } => ( - f64::from(*depth), - CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, - ), - SketchParams::CountSketch { width, depth } => ( - f64::from(*depth), - CountSketch::, RegularPath>::estimated_memory_bytes( - *depth as usize, - *width as usize, - ) as u64, - ), - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => ( - f64::from(*depth) + 1.0, - CountMinSketchWithHeap::, RegularPath>::estimated_memory_bytes( - *depth as usize, - *width as usize, - *heap_size as usize, - &heap_key, - ) as u64, - ), - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => ( - f64::from(*depth) + 1.0, - CountSketchWithHeap::, RegularPath>::estimated_memory_bytes( - *depth as usize, - *width as usize, - *heap_size as usize, - &heap_key, - ) as u64, - ), - SketchParams::Kll { k, .. } => ( - 1.0, - kll_fixed_bytes() + 8 * kll_buffer_items(*k, input_rows), - ), - // The executor builds no DDSketch outside α in (0, 1), where - // sketchlib's estimate panics: priced as an empty one. - SketchParams::DDSketch { alpha } if !(*alpha > 0.0 && *alpha < 1.0) => { - (1.0, std::mem::size_of::() as u64) - } - SketchParams::DDSketch { alpha } => ( - 1.0, - DdSketch::estimated_memory_bytes( - *alpha, - rows, - 1.0, - DDSKETCH_ASSUMED_VALUE_RANGE, - !input_rows.non_negative, - ) as u64, - ), - SketchParams::Hll { precision, .. } => ( - 1.0, - HyperLogLog::::estimated_memory_bytes(usize::from(*precision)) as u64, - ), - SketchParams::Kmv { k, .. } => (1.0, Kmv::estimated_memory_bytes(*k as usize, rows) as u64), - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => ( - univmon_expected_layers(*layers) * (f64::from(*sketch_rows) + 1.0), - UnivMon::estimated_memory_bytes( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - &DataInput::Bytes(&[0; ASSUMED_VALUE_KEY_BYTES]), - ) as u64, - ), - } -} - -/// Levels `asap_sketchlib`'s KLL has room for (`MAX_LEVELS`). -const KLL_MAX_LEVELS: usize = 61; - -/// Bytes every `asap_sketchlib` KLL state holds whatever its samples: the -/// struct and the 8-byte level-boundary array of `KLL_MAX_LEVELS + 1` entries it -/// allocates on creation (`Kll::init_internal`). 744 bytes, an empty -/// sketch's `memory_bytes()`. -fn kll_fixed_bytes() -> u64 { - (std::mem::size_of::>() + 8 * (KLL_MAX_LEVELS + 1)) as u64 -} - -/// The most items `asap_sketchlib`'s KLL holds for `k`, the capacity its item -/// array grows to (`Kll::init_internal` → `compute_max_capacity(k, 8)`): over its -/// [`KLL_MAX_LEVELS`] levels, -/// `max(ceil(k · (2/3)^i), 8)` each, with `k` capped at [`KLL_MAX_K`] as the -/// library caps it. 1,206 items at `k` = 269. -fn kll_capacity_items(k: u32) -> u64 { - const MIN_LEVEL_CAPACITY: f64 = 8.0; - let k = f64::from(k.min(KLL_MAX_K)); - let mut scale = 1.0_f64; - let mut total = 0; - for _ in 0..KLL_MAX_LEVELS { - total += (k * scale).ceil().max(MIN_LEVEL_CAPACITY) as u64; - scale *= 2.0 / 3.0; - } - total -} - -/// Items `asap_sketchlib`'s KLL for `k` allocates over `input_rows`: its -/// item array starts empty and grows when full, to at least twice its -/// length and at least 16 items (`INITIAL_BUFFER_ITEMS`), at most its full -/// capacity ([`kll_capacity_items`]). So `n` rows take the smallest -/// `16 · 2^j` ≥ `n`, capped at the capacity: exactly what the library holds -/// up to `max(k, 8)` rows, before anything compacts; 16 items for one row, -/// the capacity when the rows are unknown. Past that, compaction keeps the -/// array within this size, but the library also holds merge scratch space, -/// not modelled here, that adds up to about a fifth. A state is assumed to -/// fold at least one row, as a DDSketch's is. -fn kll_buffer_items(k: u32, input_rows: StateInputRows) -> u64 { - const INITIAL_BUFFER_ITEMS: u64 = 16; - let capacity = kll_capacity_items(k); - match input_rows.rows_per_state { - None => capacity, - Some(rows_per_state) => { - let grown = rows_per_state - .max(1) - .div_ceil(INITIAL_BUFFER_ITEMS) - .next_power_of_two(); - grown.saturating_mul(INITIAL_BUFFER_ITEMS).min(capacity) - } - } -} - -/// Bytes of the key one Count-Min / Count Sketch top-k heap slot holds: an -/// assumption, since the workload declares no label sizes. Such a key is a -/// label set in the executor's tuple-key encoding, a tag byte and a 4-byte -/// length before each value; this assumes three labels of 16 bytes, the -/// size [`row_bytes`] gives a string. -const ASSUMED_HEAP_KEY_BYTES: usize = 3 * (1 + 4 + 16); - -/// Bytes of the key one UnivMon heap slot holds: an assumption, since the -/// workload declares no label sizes. UnivMon counts the values of one -/// column, the executor keys each by its type tag and bytes, and a label -/// value is assumed to be 16 bytes, the size [`row_bytes`] gives a string. -/// A sample value's key owns no bytes, so this bounds it too. -const ASSUMED_VALUE_KEY_BYTES: usize = 1 + 16; - -/// Layers a UnivMon row updates in expectation. `asap_sketchlib` updates -/// layers `0..=b`, with bottom layer `b` the index of the first zero among -/// hash bits `1..layers`, capped at `layers - 1`; so it reaches layer `j` -/// with probability `2^-j`, and the expectation is -/// `sum_{j < layers} 2^-j = 2 - 2^(1 - layers)`. -fn univmon_expected_layers(layers: u8) -> f64 { - 2.0 - 2f64.powi(1 - i32::from(layers.max(1))) -} - -/// Ratio of the largest to the smallest value magnitude a DDSketch is -/// assumed to see: nine decades, e.g. latencies from 1 ns to 1 s, priced as -/// magnitudes in `[1, 1e9]` (its bins depend on the ratio alone). An -/// assumption, not workload evidence: the data workload declares no value -/// range. -const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::summary_type::SketchAlgorithm; - - /// An exact accumulator's state is priced at the executor's per-group - /// state, 56 bytes for every kind (a counter's state alone is 48), so a - /// tumbling-window chain of exact sums keeps 56 bytes per tumbling window and group. - #[test] - fn exact_state_is_priced_at_the_executors_state_size() { - use asap_types::summary_type::{ExactKind, ExactParams}; - for (kind, params) in [ - (ExactKind::Sum, ExactParams::Sum), - (ExactKind::Increase, ExactParams::Increase), - (ExactKind::Rate, ExactParams::Rate), - ] { - assert_eq!( - summary_shape( - &FieldDataType::ExactAggregate(kind, params), - StateInputRows::UNKNOWN - ), - (1.0, 56) - ); - } - assert_eq!(asap_types::summary_type::EXACT_ACCUMULATOR_STATE_BYTES, 56); - } - - /// The bytes `asap_sketchlib`'s DDSketch holds after adding `values`. - fn library_ddsketch_bytes(alpha: f64, values: &[f64]) -> u64 { - let mut sketch = asap_sketchlib::sketches::DdSketch::new(alpha); - for value in values { - sketch.add(value); - } - sketch.memory_bytes() as u64 - } - - /// A DDSketch is priced for what `asap_sketchlib` can hold after its - /// rows: at least what it holds for a few samples, for samples of either - /// sign, and for many samples over nine decades; a store less for values - /// declared non-negative. Many rows are priced as unknown ones. - #[test] - fn ddsketch_state_bytes_bound_the_library() { - let alpha = 0.05; - let priced = |rows_per_state, non_negative| { - sketch_shape( - &SketchParams::DDSketch { alpha }, - StateInputRows { - rows_per_state, - non_negative, - }, - ) - .1 - }; - // Ten Pareto(2) samples (scale 1), one 1 s tumbling window of a 10 Hz series. - let pareto: Vec = (0..10) - .map(|i| (1.0 - (f64::from(i) + 0.5) / 10.0).powf(-0.5)) - .collect(); - let signed: Vec = pareto.iter().map(|v| -v).chain(pareto.clone()).collect(); - let decades: Vec = (0..10_000) - .map(|i| 10f64.powf(9.0 * f64::from((i * 7_919) % 10_000) / 10_000.0)) - .collect(); - let signed_decades: Vec = decades - .iter() - .enumerate() - .map(|(i, v)| if i % 2 == 0 { -v } else { *v }) - .collect(); - for (values, non_negative) in [ - (&pareto[..], true), - (&[-2.0][..], false), - (&[2.0][..], false), - (&[-2.0, 2.0][..], false), - (&signed[..], false), - (&decades[..], true), - (&signed_decades[..], false), - ] { - let rows = Some(values.len() as u64); - assert!( - library_ddsketch_bytes(alpha, values) <= priced(rows, non_negative), - "{} values, non-negative: {non_negative}", - values.len() - ); - } - assert!(priced(Some(10_000), true) < priced(Some(10_000), false)); - assert_eq!(priced(Some(10_000), false), priced(None, false)); - } - - /// A KLL state is charged what `asap_sketchlib` allocates for the - /// samples it sees, not its full capacity: its fixed struct and level - /// array, plus the item array the library starts empty and doubles from - /// 16 items as samples arrive, up to `compute_max_capacity(k, 8)`. E.g. - /// a 10 s tumbling window of a series sampled every 15 s sees one sample: 744 + 128 - /// = 872 bytes, where the full capacity at `k` = 269 is 1,206 items. - #[test] - fn kll_state_bytes_grow_with_the_samples_it_sees() { - let k = 269; - // The library's bytes after `n` samples. - let library_bytes = |n: u32| { - let mut sketch = asap_sketchlib::Kll::::with_k(k as i32); - for i in 0..n { - sketch.update(&f64::from(i)); - } - sketch.memory_bytes() as u64 - }; - let model_bytes = |rows_per_state| { - sketch_shape( - &SketchParams::Kll { k }, - StateInputRows { - rows_per_state, - non_negative: true, - }, - ) - .1 - }; - // An empty sketch holds its fixed bytes; one sample adds 16 items. - assert_eq!(library_bytes(0), kll_fixed_bytes()); - assert_eq!(library_bytes(0), 744); - assert_eq!(library_bytes(1), 872); - // Up to `k` samples nothing compacts, so the library holds exactly - // the fixed bytes and the grown array (and no merge scratch space). - for n in [1, 4, 16, 17, 100, k] { - assert_eq!( - model_bytes(Some(u64::from(n))), - library_bytes(n), - "{n} samples" - ); - } - // Many samples are charged the full capacity, the most the array - // grows to. - assert_eq!(model_bytes(Some(5_000)), 744 + 8 * 1_206); - // A state assumed to see no sample holds at least one, as a - // DDSketch's does; unknown samples fill the full capacity. - assert_eq!(model_bytes(Some(0)), 872); - assert_eq!(model_bytes(None), 744 + 8 * 1_206); - } - - /// Plan selection prices each state the executor builds per group at - /// what the executor charges for it (#947). Each family is built - /// through the executor's factory and fed `n` rows shaped as the - /// pricing assumes: signed values over [`DDSKETCH_ASSUMED_VALUE_RANGE`], - /// label values of [`ASSUMED_VALUE_KEY_BYTES`] keys for UnivMon, or - /// keyed rows whose tuple keys are [`ASSUMED_HEAP_KEY_BYTES`] long. The - /// heap families' shapes are dominated by their heaps. The charge is - /// the sketch's bytes plus the accumulator's own fields, named per - /// family. It equals the priced bytes where `asap_sketchlib`'s estimate - /// is exact (Count-Min, Count Sketch, HLL, KMV at `n` distinct items) and - /// for KLL up to `k` rows. For DDSketch, the heap sketches and UnivMon - /// the price bounds it, by at most 1.5 times once the state is full; - /// UnivMon's price also bounds the bytes its sketch holds. - /// KLL past `k` rows is checked only loosely: the planner's formula - /// leaves out the compaction scratch space, a known gap until a - /// sketchlib estimate without the CDF cache can price KLL. - /// - /// HydraCms is left out: plan selection prices an independent Count-Min - /// per group, the executor one shared grid, so the two agree only at - /// about 79 groups for a 5 × 32 grid of 64 × 3 cells (a follow-up). - #[test] - fn state_bytes_bound_the_executors_charge() { - use asap_executor::factory::create_planner_accumulator; - use asap_executor::summary_kernels::univmon::UnivMonAccumulator; - use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; - use asap_executor::summary_kernels::HllSketchAccumulator; - use asap_executor::values::Value; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::summary_type::{ - NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, - }; - use std::mem::{align_of, size_of}; - use SketchAlgorithm as A; - /// How the priced bytes relate to the sketch's. - enum Priced { - Exact, - /// Exact up to `k` rows; past them at most the formula's - /// missing compaction scratch space over it. - Kll { - k: u32, - }, - /// At least, and at most 1.5 times once the state is full. - Bound, - } - /// The rows a family is fed. - enum Rows { - /// Signed values over the assumed range. - Values, - /// One label value a row, as Pass 1 feeds UnivMon a column. - Labels, - /// A label set and a weight a row. - Keyed, - } - // (family, parameters, rows fed, how it is priced, the - // accumulator's own fields, rows at which the state is full: four - // times the heap for the heap sketches, so each heap churns). - // - // The accumulators' own fields, beside the sketch each wraps: - // Count-Min / Count Sketch's width, depth and capacity, padded to - // the struct's alignment; HLL's precision tag; UnivMon's two key - // bounds. - let weighted_frequency_shape = - (3 * size_of::()).next_multiple_of(align_of::()); - let hll_precision_tag = size_of::() - - size_of::>(); - let univmon_key_bounds = 2 * size_of::(); - let families = [ - ( - A::Kll, - SketchParams::Kll { k: 200 }, - Rows::Values, - Priced::Kll { k: 200 }, - 0, - 10_000, - ), - ( - A::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - Rows::Values, - Priced::Bound, - 0, - 10_000, - ), - ( - A::Cms, - SketchParams::Cms { - width: 1_024, - depth: 5, - }, - Rows::Keyed, - Priced::Exact, - 0, - 10_000, - ), - ( - A::CountSketch, - SketchParams::CountSketch { - width: 1_024, - depth: 5, - }, - Rows::Keyed, - Priced::Exact, - weighted_frequency_shape, - 10_000, - ), - ( - A::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 16, - depth: 1, - heap_size: 256, - }, - Rows::Keyed, - Priced::Bound, - weighted_frequency_shape, - 1_024, - ), - ( - A::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 16, - depth: 1, - heap_size: 256, - }, - Rows::Keyed, - Priced::Bound, - weighted_frequency_shape, - 1_024, - ), - ( - A::Hll, - SketchParams::Hll { precision: 12 }, - Rows::Values, - Priced::Exact, - hll_precision_tag, - 10_000, - ), - ( - A::Kmv, - SketchParams::Kmv { k: 1_024 }, - Rows::Values, - Priced::Exact, - 0, - 10_000, - ), - // Its top layer sees one row in eight: 4 096 labels fill and - // churn every layer's heap. - ( - A::UnivMon, - SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 1, - sketch_cols: 16, - layers: 4, - }, - Rows::Labels, - Priced::Bound, - univmon_key_bounds, - 4_096, - ), - ]; - let value = SummaryUpdate::column(ColumnRef::SampleValue); - let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); - let keyed = SummaryUpdate { - item: item.clone(), - ..value.clone() - }; - let labeled = SummaryUpdate { - item, - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - // A 16-byte label value: its key has the assumed size. - let label = |i: u32| format!("{i:016}"); - assert_eq!( - Value::Utf8(label(0).into()).key().unwrap().len(), - ASSUMED_VALUE_KEY_BYTES - ); - // Three 16-byte labels: a tuple key of the assumed size. - let labels = |i: u32| { - [ - label(i), - "service-frontend".to_string(), - "region-us-east-1".to_string(), - ] - }; - let key = asap_sketchlib::TupleKey::new( - labels(0) - .into_iter() - .map(asap_sketchlib::KeyColumn::String) - .collect(), - ); - assert_eq!(key.to_bytes().unwrap().len(), ASSUMED_HEAP_KEY_BYTES); - for (algorithm, params, rows, priced_as, fields, full) in &families { - for n in [1, 100, *full] { - let family = FieldDataType::Sketch(*algorithm); - let input = match rows { - Rows::Values => &value, - Rows::Labels => &labeled, - Rows::Keyed => &keyed, - }; - let mut updater = create_planner_accumulator(&family, Some(params), input).unwrap(); - for i in 0..n { - match rows { - // Distinct magnitudes over the assumed range, in a - // scrambled order, alternating in sign. - Rows::Values => { - let at = f64::from(i * 7_919 % n) / f64::from(n); - let magnitude = DDSKETCH_ASSUMED_VALUE_RANGE.powf(at); - let sign = if i % 2 == 0 { 1.0 } else { -1.0 }; - updater.update_single(sign * magnitude, 0).unwrap(); - } - Rows::Labels => updater - .update_value(&Value::Utf8(label(i).into()), 0) - .unwrap(), - Rows::Keyed => updater.update_keyed(&labels(i), 1.0, 0).unwrap(), - } - } - let state = updater.into_accumulator(); - let charged = state.approx_memory_bytes(); - let priced = fields - + sketch_shape( - params, - StateInputRows { - rows_per_state: Some(u64::from(n)), - non_negative: false, - }, - ) - .1 as usize; - let context = format!("{algorithm:?} after {n} rows: {priced} priced"); - match priced_as { - Priced::Exact => assert_eq!(charged, priced, "{context}"), - Priced::Kll { k } if n <= *k => assert_eq!(charged, priced, "{context}"), - // sketchlib bounds KLL's compaction scratch space at - // 2 (k - 1) items. - Priced::Kll { k } => assert!( - charged <= priced + 8 * 2 * (*k as usize - 1), - "{context}, {charged} charged" - ), - Priced::Bound => { - assert!(charged <= priced, "{context}, {charged} charged"); - if n == *full { - assert!( - priced as f64 <= 1.5 * charged as f64, - "{context}, {charged} charged" - ); - } - } - } - // UnivMon charges sketchlib's estimate itself, since its - // `memory_bytes` walks the heaps: the sketch holds no more. - // The estimate is a heap whose digest index has grown once - // more under churn, which the test's labels do not reach: a - // full heap holds about two thirds of it (1.52 times less - // here), so this bound is 1.6 times rather than 1.5. - if let Some(univmon) = state.as_any().downcast_ref::() { - let held = fields + univmon.sketch().memory_bytes(); - assert!(held <= charged, "{context}, {held} held"); - if n == *full { - assert!(priced as f64 <= 1.6 * held as f64, "{context}, {held} held"); - } - } - } - } - } -} diff --git a/crates/cost/src/node/state_bytes/mod.rs b/crates/cost/src/node/state_bytes/mod.rs new file mode 100644 index 000000000..c0ca6c042 --- /dev/null +++ b/crates/cost/src/node/state_bytes/mod.rs @@ -0,0 +1,274 @@ +//! The bytes one summary or exact state holds and its update operations +//! per input row, from its parameters and the rows it folds. + +use asap_types::ir::operator::AggIntent; +use asap_types::ir::schema::{DataType, FieldDataType, Schema}; +use asap_types::summary_type::KLL_MAX_K; +use asap_types::summary_type::{SketchParams, EXACT_ACCUMULATOR_STATE_BYTES}; + +/// Plain values are 8 bytes, strings 16; summary columns are sized apart. +pub fn row_bytes(schema: &Schema) -> u64 { + schema + .fields + .iter() + .map(|f| match &f.dtype { + FieldDataType::Plain(DataType::Utf8) => 16, + _ => 8, + }) + .sum::() + .max(1) +} + +/// Exact state that keeps every value or item of its group, so grows with +/// the data; other exact accumulators are fixed-size. +pub fn keeps_every_value(measure: &AggIntent) -> bool { + matches!( + measure, + AggIntent::Quantile { .. } + | AggIntent::TopK { .. } + | AggIntent::Cardinality { .. } + | AggIntent::FrequencyL2 { .. } + | AggIntent::FrequencyEntropy { .. } + ) +} + +/// Update operations per input row and bytes per state, from the sketch's +/// parameters and `input_rows` (the rows one state folds). The bytes are +/// `asap_sketchlib`'s `estimated_memory_bytes` for the type the executor +/// builds (Count-Min and Count Sketch over `f64` counters), given what the +/// state is assumed to hold: +/// - a Count-Min / Count Sketch top-k heap full of keys of +/// `ASSUMED_HEAP_KEY_BYTES`; each UnivMon layer's heap full of keys of +/// `ASSUMED_VALUE_KEY_BYTES`; +/// - DDSketch and KMV: the rows the state folds, at least one (any number +/// when unknown), each a distinct item for KMV; a DDSketch's values span +/// `DDSKETCH_ASSUMED_VALUE_RANGE` and fill its negative store too +/// unless proven non-negative. +/// +/// KLL is priced without the library's estimate: its fixed bytes +/// (`kll_fixed_bytes`: the struct and its level array) plus the 8-byte +/// item array as `asap_sketchlib` grows it for the rows the state folds, at +/// most its full capacity (`kll_buffer_items`). +/// +/// An exact accumulator's state is the executor's per-group state, +/// [`EXACT_ACCUMULATOR_STATE_BYTES`] for every kind. +/// +/// Update operations per input row: one counter per row of depth for +/// Count-Min and Count-Sketch, plus one heap update for their top-k variants; +/// for UnivMon, the same per layer it updates (`univmon_expected_layers`); +/// 1 for every other sketch. +pub fn summary_shape(family: &FieldDataType, input_rows: StateInputRows) -> (f64, u64) { + match family { + FieldDataType::Sketch(..) => { + let _ = input_rows; + todo!("#1207 PR 6: the state shape of θ from the candidate's LogicalProperties::parameters") + } + FieldDataType::ExactAggregate(..) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), + _ => (1.0, 8), + } +} + +/// What one summary state's bytes depend on besides its parameters (only +/// a KLL's, a DDSketch's and a KMV's do): the input rows it folds and the +/// sign of their values. +/// The rows are whatever its producer emits (raw data samples, rates, sums), +/// not a summary sample. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct StateInputRows { + /// Input rows one state folds, the average over its states rounded up; + /// `None` when unknown, which sizes it for every value it can be assumed + /// to see. + pub rows_per_state: Option, + /// Whether every value folded is proven non-negative (raw data samples of + /// sources declared non-negative), so a DDSketch fills no negative store. + pub non_negative: bool, +} + +impl StateInputRows { + /// Unknown rows of unknown sign: the most a state is assumed to hold. + pub const UNKNOWN: Self = Self { + rows_per_state: None, + non_negative: false, + }; +} + +/// [`summary_shape`] of one sketch: its update operations per row and its +/// bytes per state. With [`StateInputRows::UNKNOWN`], the bytes are also the +/// size the measured table interpolates between +/// (`crate::measured`). +pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, u64) { + use asap_sketchlib::{ + Classic, CountMinF64, CountMinSketchWithHeap, CountSketch, CountSketchWithHeap, DataInput, + DdSketch, HyperLogLog, Kmv, RegularPath, UnivMon, Vector2D, + }; + let heap_key = DataInput::Bytes(&[0; ASSUMED_HEAP_KEY_BYTES]); + let rows = input_rows.rows_per_state.map_or(usize::MAX, |rows| { + usize::try_from(rows.max(1)).unwrap_or(usize::MAX) + }); + match params { + SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), + SketchParams::Cms { width, depth } => ( + f64::from(*depth), + CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, + ), + SketchParams::CountSketch { width, depth } => ( + f64::from(*depth), + CountSketch::, RegularPath>::estimated_memory_bytes( + *depth as usize, + *width as usize, + ) as u64, + ), + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } => ( + f64::from(*depth) + 1.0, + CountMinSketchWithHeap::, RegularPath>::estimated_memory_bytes( + *depth as usize, + *width as usize, + *heap_size as usize, + &heap_key, + ) as u64, + ), + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => ( + f64::from(*depth) + 1.0, + CountSketchWithHeap::, RegularPath>::estimated_memory_bytes( + *depth as usize, + *width as usize, + *heap_size as usize, + &heap_key, + ) as u64, + ), + SketchParams::Kll { k, .. } => ( + 1.0, + kll_fixed_bytes() + 8 * kll_buffer_items(*k, input_rows), + ), + // The executor builds no DDSketch outside α in (0, 1), where + // sketchlib's estimate panics: priced as an empty one. + SketchParams::DDSketch { alpha } if !(*alpha > 0.0 && *alpha < 1.0) => { + (1.0, std::mem::size_of::() as u64) + } + SketchParams::DDSketch { alpha } => ( + 1.0, + DdSketch::estimated_memory_bytes( + *alpha, + rows, + 1.0, + DDSKETCH_ASSUMED_VALUE_RANGE, + !input_rows.non_negative, + ) as u64, + ), + SketchParams::Hll { precision, .. } => ( + 1.0, + HyperLogLog::::estimated_memory_bytes(usize::from(*precision)) as u64, + ), + SketchParams::Kmv { k, .. } => (1.0, Kmv::estimated_memory_bytes(*k as usize, rows) as u64), + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + .. + } => ( + univmon_expected_layers(*layers) * (f64::from(*sketch_rows) + 1.0), + UnivMon::estimated_memory_bytes( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + usize::from(*layers), + &DataInput::Bytes(&[0; ASSUMED_VALUE_KEY_BYTES]), + ) as u64, + ), + } +} + +/// Levels `asap_sketchlib`'s KLL has room for (`MAX_LEVELS`). +const KLL_MAX_LEVELS: usize = 61; + +/// Bytes every `asap_sketchlib` KLL state holds whatever its samples: the +/// struct and the 8-byte level-boundary array of `KLL_MAX_LEVELS + 1` entries it +/// allocates on creation (`Kll::init_internal`). 744 bytes, an empty +/// sketch's `memory_bytes()`. +fn kll_fixed_bytes() -> u64 { + (std::mem::size_of::>() + 8 * (KLL_MAX_LEVELS + 1)) as u64 +} + +/// The most items `asap_sketchlib`'s KLL holds for `k`, the capacity its item +/// array grows to (`Kll::init_internal` → `compute_max_capacity(k, 8)`): over its +/// [`KLL_MAX_LEVELS`] levels, +/// `max(ceil(k · (2/3)^i), 8)` each, with `k` capped at [`KLL_MAX_K`] as the +/// library caps it. 1,206 items at `k` = 269. +fn kll_capacity_items(k: u32) -> u64 { + const MIN_LEVEL_CAPACITY: f64 = 8.0; + let k = f64::from(k.min(KLL_MAX_K)); + let mut scale = 1.0_f64; + let mut total = 0; + for _ in 0..KLL_MAX_LEVELS { + total += (k * scale).ceil().max(MIN_LEVEL_CAPACITY) as u64; + scale *= 2.0 / 3.0; + } + total +} + +/// Items `asap_sketchlib`'s KLL for `k` allocates over `input_rows`: its +/// item array starts empty and grows when full, to at least twice its +/// length and at least 16 items (`INITIAL_BUFFER_ITEMS`), at most its full +/// capacity ([`kll_capacity_items`]). So `n` rows take the smallest +/// `16 · 2^j` ≥ `n`, capped at the capacity: exactly what the library holds +/// up to `max(k, 8)` rows, before anything compacts; 16 items for one row, +/// the capacity when the rows are unknown. Past that, compaction keeps the +/// array within this size, but the library also holds merge scratch space, +/// not modelled here, that adds up to about a fifth. A state is assumed to +/// fold at least one row, as a DDSketch's is. +fn kll_buffer_items(k: u32, input_rows: StateInputRows) -> u64 { + const INITIAL_BUFFER_ITEMS: u64 = 16; + let capacity = kll_capacity_items(k); + match input_rows.rows_per_state { + None => capacity, + Some(rows_per_state) => { + let grown = rows_per_state + .max(1) + .div_ceil(INITIAL_BUFFER_ITEMS) + .next_power_of_two(); + grown.saturating_mul(INITIAL_BUFFER_ITEMS).min(capacity) + } + } +} + +/// Bytes of the key one Count-Min / Count Sketch top-k heap slot holds: an +/// assumption, since the workload declares no label sizes. Such a key is a +/// label set in the executor's tuple-key encoding, a tag byte and a 4-byte +/// length before each value; this assumes three labels of 16 bytes, the +/// size [`row_bytes`] gives a string. +const ASSUMED_HEAP_KEY_BYTES: usize = 3 * (1 + 4 + 16); + +/// Bytes of the key one UnivMon heap slot holds: an assumption, since the +/// workload declares no label sizes. UnivMon counts the values of one +/// column, the executor keys each by its type tag and bytes, and a label +/// value is assumed to be 16 bytes, the size [`row_bytes`] gives a string. +/// A sample value's key owns no bytes, so this bounds it too. +const ASSUMED_VALUE_KEY_BYTES: usize = 1 + 16; + +/// Layers a UnivMon row updates in expectation. `asap_sketchlib` updates +/// layers `0..=b`, with bottom layer `b` the index of the first zero among +/// hash bits `1..layers`, capped at `layers - 1`; so it reaches layer `j` +/// with probability `2^-j`, and the expectation is +/// `sum_{j < layers} 2^-j = 2 - 2^(1 - layers)`. +fn univmon_expected_layers(layers: u8) -> f64 { + 2.0 - 2f64.powi(1 - i32::from(layers.max(1))) +} + +/// Ratio of the largest to the smallest value magnitude a DDSketch is +/// assumed to see: nine decades, e.g. latencies from 1 ns to 1 s, priced as +/// magnitudes in `[1, 1e9]` (its bins depend on the ratio alone). An +/// assumption, not workload evidence: the data workload declares no value +/// range. +const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; + +#[cfg(test)] +mod tests; diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs new file mode 100644 index 000000000..131a9f26f --- /dev/null +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -0,0 +1,395 @@ +use super::*; +use asap_types::summary_type::SketchAlgorithm; + +/// An exact accumulator's state is priced at the executor's per-group +/// state, 56 bytes for every kind (a counter's state alone is 48), so a +/// tumbling-window chain of exact sums keeps 56 bytes per tumbling window and group. +#[test] +fn exact_state_is_priced_at_the_executors_state_size() { + use asap_types::summary_type::{ExactKind, ExactParams}; + for (kind, params) in [ + (ExactKind::Sum, ExactParams::Sum), + (ExactKind::Increase, ExactParams::Increase), + (ExactKind::Rate, ExactParams::Rate), + ] { + assert_eq!( + summary_shape( + &FieldDataType::ExactAggregate(kind, params), + StateInputRows::UNKNOWN + ), + (1.0, 56) + ); + } + assert_eq!(asap_types::summary_type::EXACT_ACCUMULATOR_STATE_BYTES, 56); +} + +/// The bytes `asap_sketchlib`'s DDSketch holds after adding `values`. +fn library_ddsketch_bytes(alpha: f64, values: &[f64]) -> u64 { + let mut sketch = asap_sketchlib::sketches::DdSketch::new(alpha); + for value in values { + sketch.add(value); + } + sketch.memory_bytes() as u64 +} + +/// A DDSketch is priced for what `asap_sketchlib` can hold after its +/// rows: at least what it holds for a few samples, for samples of either +/// sign, and for many samples over nine decades; a store less for values +/// declared non-negative. Many rows are priced as unknown ones. +#[test] +fn ddsketch_state_bytes_bound_the_library() { + let alpha = 0.05; + let priced = |rows_per_state, non_negative| { + sketch_shape( + &SketchParams::DDSketch { alpha }, + StateInputRows { + rows_per_state, + non_negative, + }, + ) + .1 + }; + // Ten Pareto(2) samples (scale 1), one 1 s tumbling window of a 10 Hz series. + let pareto: Vec = (0..10) + .map(|i| (1.0 - (f64::from(i) + 0.5) / 10.0).powf(-0.5)) + .collect(); + let signed: Vec = pareto.iter().map(|v| -v).chain(pareto.clone()).collect(); + let decades: Vec = (0..10_000) + .map(|i| 10f64.powf(9.0 * f64::from((i * 7_919) % 10_000) / 10_000.0)) + .collect(); + let signed_decades: Vec = decades + .iter() + .enumerate() + .map(|(i, v)| if i % 2 == 0 { -v } else { *v }) + .collect(); + for (values, non_negative) in [ + (&pareto[..], true), + (&[-2.0][..], false), + (&[2.0][..], false), + (&[-2.0, 2.0][..], false), + (&signed[..], false), + (&decades[..], true), + (&signed_decades[..], false), + ] { + let rows = Some(values.len() as u64); + assert!( + library_ddsketch_bytes(alpha, values) <= priced(rows, non_negative), + "{} values, non-negative: {non_negative}", + values.len() + ); + } + assert!(priced(Some(10_000), true) < priced(Some(10_000), false)); + assert_eq!(priced(Some(10_000), false), priced(None, false)); +} + +/// A KLL state is charged what `asap_sketchlib` allocates for the +/// samples it sees, not its full capacity: its fixed struct and level +/// array, plus the item array the library starts empty and doubles from +/// 16 items as samples arrive, up to `compute_max_capacity(k, 8)`. E.g. +/// a 10 s tumbling window of a series sampled every 15 s sees one sample: 744 + 128 +/// = 872 bytes, where the full capacity at `k` = 269 is 1,206 items. +#[test] +fn kll_state_bytes_grow_with_the_samples_it_sees() { + let k = 269; + // The library's bytes after `n` samples. + let library_bytes = |n: u32| { + let mut sketch = asap_sketchlib::Kll::::with_k(k as i32); + for i in 0..n { + sketch.update(&f64::from(i)); + } + sketch.memory_bytes() as u64 + }; + let model_bytes = |rows_per_state| { + sketch_shape( + &SketchParams::Kll { k }, + StateInputRows { + rows_per_state, + non_negative: true, + }, + ) + .1 + }; + // An empty sketch holds its fixed bytes; one sample adds 16 items. + assert_eq!(library_bytes(0), kll_fixed_bytes()); + assert_eq!(library_bytes(0), 744); + assert_eq!(library_bytes(1), 872); + // Up to `k` samples nothing compacts, so the library holds exactly + // the fixed bytes and the grown array (and no merge scratch space). + for n in [1, 4, 16, 17, 100, k] { + assert_eq!( + model_bytes(Some(u64::from(n))), + library_bytes(n), + "{n} samples" + ); + } + // Many samples are charged the full capacity, the most the array + // grows to. + assert_eq!(model_bytes(Some(5_000)), 744 + 8 * 1_206); + // A state assumed to see no sample holds at least one, as a + // DDSketch's does; unknown samples fill the full capacity. + assert_eq!(model_bytes(Some(0)), 872); + assert_eq!(model_bytes(None), 744 + 8 * 1_206); +} + +/// Plan selection prices each state the executor builds per group at +/// what the executor charges for it (#947). Each family is built +/// through the executor's factory and fed `n` rows shaped as the +/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_VALUE_RANGE`], +/// label values of [`ASSUMED_VALUE_KEY_BYTES`] keys for UnivMon, or +/// keyed rows whose tuple keys are [`ASSUMED_HEAP_KEY_BYTES`] long. The +/// heap families' shapes are dominated by their heaps. The charge is +/// the sketch's bytes plus the accumulator's own fields, named per +/// family. It equals the priced bytes where `asap_sketchlib`'s estimate +/// is exact (Count-Min, Count Sketch, HLL, KMV at `n` distinct items) and +/// for KLL up to `k` rows. For DDSketch, the heap sketches and UnivMon +/// the price bounds it, by at most 1.5 times once the state is full; +/// UnivMon's price also bounds the bytes its sketch holds. +/// KLL past `k` rows is checked only loosely: the planner's formula +/// leaves out the compaction scratch space, a known gap until a +/// sketchlib estimate without the CDF cache can price KLL. +/// +/// HydraCms is left out: plan selection prices an independent Count-Min +/// per group, the executor one shared grid, so the two agree only at +/// about 79 groups for a 5 × 32 grid of 64 × 3 cells (a follow-up). +#[test] +fn state_bytes_bound_the_executors_charge() { + use asap_executor::factory::create_planner_accumulator; + use asap_executor::summary_kernels::univmon::UnivMonAccumulator; + use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; + use asap_executor::summary_kernels::HllSketchAccumulator; + use asap_executor::values::Value; + use asap_types::ir::scalar::expr_ir::ColumnRef; + use asap_types::summary_type::{SummaryInputExpr, SummaryUpdate}; + use std::mem::{align_of, size_of}; + use SketchAlgorithm as A; + /// How the priced bytes relate to the sketch's. + enum Priced { + Exact, + /// Exact up to `k` rows; past them at most the formula's + /// missing compaction scratch space over it. + Kll { + k: u32, + }, + /// At least, and at most 1.5 times once the state is full. + Bound, + } + /// The rows a family is fed. + enum Rows { + /// Signed values over the assumed range. + Values, + /// One label value a row, as Pass 1 feeds UnivMon a column. + Labels, + /// A label set and a weight a row. + Keyed, + } + // (family, parameters, rows fed, how it is priced, the + // accumulator's own fields, rows at which the state is full: four + // times the heap for the heap sketches, so each heap churns). + // + // The accumulators' own fields, beside the sketch each wraps: + // Count-Min / Count Sketch's width, depth and capacity, padded to + // the struct's alignment; HLL's precision tag; UnivMon's two key + // bounds. + let weighted_frequency_shape = + (3 * size_of::()).next_multiple_of(align_of::()); + let hll_precision_tag = size_of::() + - size_of::>(); + let univmon_key_bounds = 2 * size_of::(); + let families = [ + ( + A::Kll, + SketchParams::Kll { k: 200 }, + Rows::Values, + Priced::Kll { k: 200 }, + 0, + 10_000, + ), + ( + A::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + Rows::Values, + Priced::Bound, + 0, + 10_000, + ), + ( + A::Cms, + SketchParams::Cms { + width: 1_024, + depth: 5, + }, + Rows::Keyed, + Priced::Exact, + 0, + 10_000, + ), + ( + A::CountSketch, + SketchParams::CountSketch { + width: 1_024, + depth: 5, + }, + Rows::Keyed, + Priced::Exact, + weighted_frequency_shape, + 10_000, + ), + ( + A::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 16, + depth: 1, + heap_size: 256, + }, + Rows::Keyed, + Priced::Bound, + weighted_frequency_shape, + 1_024, + ), + ( + A::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: 16, + depth: 1, + heap_size: 256, + }, + Rows::Keyed, + Priced::Bound, + weighted_frequency_shape, + 1_024, + ), + ( + A::Hll, + SketchParams::Hll { precision: 12 }, + Rows::Values, + Priced::Exact, + hll_precision_tag, + 10_000, + ), + ( + A::Kmv, + SketchParams::Kmv { k: 1_024 }, + Rows::Values, + Priced::Exact, + 0, + 10_000, + ), + // Its top layer sees one row in eight: 4 096 labels fill and + // churn every layer's heap. + ( + A::UnivMon, + SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 1, + sketch_cols: 16, + layers: 4, + }, + Rows::Labels, + Priced::Bound, + univmon_key_bounds, + 4_096, + ), + ]; + let value = SummaryUpdate::column(ColumnRef::SampleValue); + let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); + let keyed = SummaryUpdate { + item: item.clone(), + ..value.clone() + }; + let labeled = SummaryUpdate { + item, + weight: SummaryInputExpr::Constant(1.0), + }; + // A 16-byte label value: its key has the assumed size. + let label = |i: u32| format!("{i:016}"); + assert_eq!( + Value::Utf8(label(0).into()).key().unwrap().len(), + ASSUMED_VALUE_KEY_BYTES + ); + // Three 16-byte labels: a tuple key of the assumed size. + let labels = |i: u32| { + [ + label(i), + "service-frontend".to_string(), + "region-us-east-1".to_string(), + ] + }; + let key = asap_sketchlib::TupleKey::new( + labels(0) + .into_iter() + .map(asap_sketchlib::KeyColumn::String) + .collect(), + ); + assert_eq!(key.to_bytes().unwrap().len(), ASSUMED_HEAP_KEY_BYTES); + for (algorithm, params, rows, priced_as, fields, full) in &families { + for n in [1, 100, *full] { + let family = FieldDataType::Sketch(*algorithm); + let input = match rows { + Rows::Values => &value, + Rows::Labels => &labeled, + Rows::Keyed => &keyed, + }; + let mut updater = create_planner_accumulator(&family, Some(params), input).unwrap(); + for i in 0..n { + match rows { + // Distinct magnitudes over the assumed range, in a + // scrambled order, alternating in sign. + Rows::Values => { + let at = f64::from(i * 7_919 % n) / f64::from(n); + let magnitude = DDSKETCH_ASSUMED_VALUE_RANGE.powf(at); + let sign = if i % 2 == 0 { 1.0 } else { -1.0 }; + updater.update_single(sign * magnitude, 0).unwrap(); + } + Rows::Labels => updater + .update_value(&Value::Utf8(label(i).into()), 0) + .unwrap(), + Rows::Keyed => updater.update_keyed(&labels(i), 1.0, 0).unwrap(), + } + } + let state = updater.into_accumulator(); + let charged = state.approx_memory_bytes(); + let priced = fields + + sketch_shape( + params, + StateInputRows { + rows_per_state: Some(u64::from(n)), + non_negative: false, + }, + ) + .1 as usize; + let context = format!("{algorithm:?} after {n} rows: {priced} priced"); + match priced_as { + Priced::Exact => assert_eq!(charged, priced, "{context}"), + Priced::Kll { k } if n <= *k => assert_eq!(charged, priced, "{context}"), + // sketchlib bounds KLL's compaction scratch space at + // 2 (k - 1) items. + Priced::Kll { k } => assert!( + charged <= priced + 8 * 2 * (*k as usize - 1), + "{context}, {charged} charged" + ), + Priced::Bound => { + assert!(charged <= priced, "{context}, {charged} charged"); + if n == *full { + assert!( + priced as f64 <= 1.5 * charged as f64, + "{context}, {charged} charged" + ); + } + } + } + // UnivMon charges sketchlib's estimate itself, since its + // `memory_bytes` walks the heaps: the sketch holds no more. + // The estimate is a heap whose digest index has grown once + // more under churn, which the test's labels do not reach: a + // full heap holds about two thirds of it (1.52 times less + // here), so this bound is 1.6 times rather than 1.5. + if let Some(univmon) = state.as_any().downcast_ref::() { + let held = fields + univmon.sketch().memory_bytes(); + assert!(held <= charged, "{context}, {held} held"); + if n == *full { + assert!(priced as f64 <= 1.6 * held as f64, "{context}, {held} held"); + } + } + } + } +} diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json index 5cb411f72..fae02b115 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json @@ -939,9 +939,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1163,9 +1160,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json index 4d42935a1..35e9d5196 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json @@ -1112,9 +1112,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1324,9 +1321,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -1598,9 +1592,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1794,9 +1785,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -2010,9 +1998,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity" diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json index 8cca94ae9..934e7a5d6 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json @@ -787,9 +787,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1011,9 +1008,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -1250,9 +1244,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity" diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json index 33b444675..657fc6322 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json @@ -534,9 +534,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -692,9 +689,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity" diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json index a491b2f63..d2401d112 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json @@ -7469,9 +7469,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -7681,9 +7678,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -8036,9 +8030,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -8232,9 +8223,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json index 2e30b8d58..07f84f744 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json @@ -7244,9 +7244,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -7456,9 +7453,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -7730,9 +7724,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -7926,9 +7917,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json index 3adc6ca62..db2d86cae 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json @@ -425,9 +425,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json index 3ffc88b31..976174aee 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json @@ -969,9 +969,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1165,9 +1162,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json index e0a83a8d0..2301ac533 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json @@ -824,9 +824,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1020,9 +1017,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json index 5156b5315..25a992051 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json @@ -754,9 +754,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -950,9 +947,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json index c69ea0939..7f7397165 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json @@ -5564,9 +5564,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -5776,9 +5773,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -6050,9 +6044,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -6246,9 +6237,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json index fbdeba67f..4cd80d8e8 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json @@ -2773,9 +2773,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -2953,9 +2950,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json index e87f425a7..8d553aae8 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json @@ -425,9 +425,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json index f27b75631..8b2451a16 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json +++ b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json @@ -710,9 +710,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity" diff --git a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json index 54a2d6f95..edf25adfe 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json @@ -2971,9 +2971,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json index 37361b98d..37a58c734 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json @@ -2091,9 +2091,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -2449,9 +2446,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -2667,9 +2661,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json index 6583082f6..cd00c9c3c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json @@ -1632,9 +1632,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json index d569fff3a..24b68f9ce 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json @@ -12351,9 +12351,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -12709,9 +12706,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -12927,9 +12921,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json index 95d124a1c..767b3e159 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json @@ -6683,9 +6683,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -7041,9 +7038,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -7259,9 +7253,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -7938,9 +7929,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json index 94d046d27..06e307546 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json @@ -7443,9 +7443,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -7801,9 +7798,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -8019,9 +8013,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": { @@ -8698,9 +8689,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", diff --git a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json index 266118af6..dda03f31b 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json @@ -3059,9 +3059,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", diff --git a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json index fa094b72a..ee5c5b6b1 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json @@ -2971,9 +2971,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", diff --git a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json index 3d269fb2c..12caf51a6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json @@ -1750,9 +1750,6 @@ "item": null, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, "reduction": "PerEntity", @@ -1968,10 +1965,6 @@ }, "weight": { "Column": "SampleValue" - }, - "weight_domain": { - "kind": "counter_derivative_of_metric", - "metric": "data_total" } }, "reduction": { diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability/mod.rs similarity index 84% rename from crates/executor/src/capability.rs rename to crates/executor/src/capability/mod.rs index 5b5ef3300..83ce44d83 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability/mod.rs @@ -54,22 +54,19 @@ pub fn validate_summary_kernel( } /// The only Hydra algorithm with a kernel: Hydra over Count-Min. Each update -/// adds one item column's value to its group, with weight 1 (a row count, -/// `UnitCount`), a positive integer constant (`PositiveConstant`) or a weight -/// column proven non-negative (a Count-Min point count only overestimates -/// when no weight is negative). +/// adds one item column's value to its group, with weight 1 (a row count), a +/// positive integer constant or a weight column. The planner offers a +/// Count-Min build only over weights proven non-negative (a Count-Min point +/// count only overestimates when no weight is negative), and the build +/// rejects a negative weight value. fn validate_hydra_cms(family: &SummaryFamilyType, input: &SummaryUpdate) -> Result<(), String> { - use planner_types::summary_type::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + use planner_types::summary_type::SummaryInputExpr; hydra_cms_shape(family)?; if !matches!(input.item, Some(SummaryInputExpr::Column(_))) { return Err("HydraCms requires one item column".into()); } - let WeightDomain::NonNegative { proof } = &input.weight_domain else { - return Err("HydraCms weights must be non-negative; the weight's sign is unknown".into()); - }; - match (&input.weight, proof) { - (SummaryInputExpr::Constant(w), NonNegativeWeightProof::UnitCount) if *w == 1.0 => Ok(()), - (SummaryInputExpr::Constant(w), NonNegativeWeightProof::PositiveConstant) => { + match &input.weight { + SummaryInputExpr::Constant(w) => { let weight = constant_row_weight(*w).map_err(|error| format!("HydraCms {error}"))?; i32::try_from(weight.get()).map(|_| ()).map_err(|_| { format!( @@ -78,11 +75,9 @@ fn validate_hydra_cms(family: &SummaryFamilyType, input: &SummaryUpdate) -> Resu ) }) } - (SummaryInputExpr::Column(_), _) => Ok(()), + SummaryInputExpr::Column(_) => Ok(()), _ => Err( - "HydraCms weight must be the unit count, a positive constant or a non-negative \ - weight column" - .into(), + "HydraCms weight must be the unit count, a positive constant or a weight column".into(), ), } } @@ -117,10 +112,9 @@ pub(crate) fn hydra_cms_shape( /// none of the query's columns for the filter predicate to read, so only a /// query-time build can apply it. The ingestion-time compiler and the /// binding of any ingestion-timed `SummaryAgg` apply this check. The planner -/// applies the same rule: Stage 2 never offers materializing at ingestion time for a -/// filtered build (`IngestionPruneReason::Filtered`), and materialization -/// timing rejects an explicit one (`FilteredSummaryAtIngestion`). This check -/// guards plans that did not come from the planner. +/// offers a filtered build at ingestion time when its input is produced at +/// ingestion time; this check refuses it until the executor feeds such a +/// build its input rows. pub fn validate_ingestion_time_summary( filter: Option<&planner_types::ir::Predicate>, ) -> Result<(), String> { @@ -148,15 +142,13 @@ fn valid_matrix(width: u32, depth: u32) -> bool { pub(crate) fn unit_frequency_column( update: &planner_types::summary_type::SummaryUpdate, ) -> Option<&planner_types::ir::scalar::ColumnRef> { - use planner_types::summary_type::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; - match (&update.item, &update.weight, &update.weight_domain) { - ( - Some(SummaryInputExpr::Column(column)), - SummaryInputExpr::Constant(weight), - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - ) if *weight == 1.0 => Some(column), + use planner_types::summary_type::SummaryInputExpr; + match (&update.item, &update.weight) { + (Some(SummaryInputExpr::Column(column)), SummaryInputExpr::Constant(weight)) + if *weight == 1.0 => + { + Some(column) + } _ => None, } } @@ -300,16 +292,4 @@ pub fn validate_exact_evaluation( } #[cfg(test)] -mod tests { - use super::*; - - // Only a query-time build applies a summary filter. - #[test] - fn filtered_summary_runs_only_at_query_time() { - let filter = planner_types::ir::Predicate::(planner_types::ir::ScalarExpr::Literal( - planner_types::ir::scalar::ScalarValue::Boolean(true), - )); - assert!(validate_ingestion_time_summary(Some(&filter)).is_err()); - assert!(validate_ingestion_time_summary::(None).is_ok()); - } -} +mod tests; diff --git a/crates/executor/src/capability/tests.rs b/crates/executor/src/capability/tests.rs new file mode 100644 index 000000000..eb54f5975 --- /dev/null +++ b/crates/executor/src/capability/tests.rs @@ -0,0 +1,11 @@ +use super::*; + +// Only a query-time build applies a summary filter. +#[test] +fn filtered_summary_runs_only_at_query_time() { + let filter = planner_types::ir::Predicate::(planner_types::ir::ScalarExpr::Literal( + planner_types::ir::scalar::ScalarValue::Boolean(true), + )); + assert!(validate_ingestion_time_summary(Some(&filter)).is_err()); + assert!(validate_ingestion_time_summary::(None).is_ok()); +} diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 9c6617d90..31a53f9fd 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -1326,14 +1326,6 @@ fn summary_agg_build( let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); }; - if matches!(family, FieldDataType::Sketch(kind) if kind == &planner_types::summary_type::SketchAlgorithm::CmsWithHeap) - && !matches!( - update.weight_domain, - planner_types::summary_type::WeightDomain::NonNegative { .. } - ) - { - return Err(invalid("CMS requires a nonnegative weight contract")); - } fn columns( expr: &SummaryInputExpr, input: &SchemaRef, diff --git a/crates/executor/src/summary_kernels/factory/ddsketch_domain.rs b/crates/executor/src/summary_kernels/factory/ddsketch_domain.rs new file mode 100644 index 000000000..57db0d5d6 --- /dev/null +++ b/crates/executor/src/summary_kernels/factory/ddsketch_domain.rs @@ -0,0 +1,59 @@ +use super::*; + +/// A DDSketch takes zero and negative samples (its zero bucket and +/// negative store) as well as positive ones; it refuses NaN, ±Inf, and +/// magnitudes outside its indexable range, including a nonzero magnitude +/// below it, which would lose its relative error in the zero bucket. +#[test] +fn accepts_zero_and_signed_samples_within_its_indexable_range() { + let updater = DDSketchAccumulatorUpdater::new(0.01); + let (minimum, maximum) = asap_sketchlib::sketches::ddsketch::ddsketch_indexable_bounds(0.01); + for value in [0.0, -0.0, 3.0, -3.0, minimum, -minimum, maximum, -maximum] { + assert!(updater.validate_single_input(value).is_ok(), "{value}"); + } + for value in [ + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + minimum / 2.0, + -minimum / 2.0, + maximum * 2.0, + -maximum * 2.0, + ] { + assert!(updater.validate_single_input(value).is_err(), "{value}"); + } +} + +/// Quantiles over 0, 0, 3, 5 and over signed samples are within alpha of +/// PromQL's interpolated quantile, at rank q (n − 1) between adjacent +/// order statistics: here each pair of neighbours shares a sign or +/// includes zero. +#[test] +fn quantiles_over_zero_and_signed_samples() { + for (samples, expected) in [ + ( + vec![0.0, 0.0, 3.0, 5.0], + [(0.25, 0.0), (0.5, 1.5), (0.75, 3.5), (1.0, 5.0)], + ), + ( + vec![-7.0, -3.0, 0.0, 4.0], + [(0.25, -4.0), (0.5, -1.5), (0.75, 1.0), (1.0, 4.0)], + ), + ] { + let mut updater = DDSketchAccumulatorUpdater::new(0.01); + for value in &samples { + updater.validate_single_input(*value).unwrap(); + updater.update_single(*value, 0).unwrap(); + } + for (q, exact) in expected { + let estimate = updater + .acc + .estimate(&planner_types::summary_type::SketchStatistic::Quantile { q }) + .unwrap(); + assert!( + (estimate - exact).abs() <= 0.01 * f64::abs(exact), + "{samples:?} q={q}: {estimate} vs {exact}" + ); + } + } +} diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory/mod.rs similarity index 81% rename from crates/executor/src/summary_kernels/factory.rs rename to crates/executor/src/summary_kernels/factory/mod.rs index 1ecea9d0d..30c087e2e 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory/mod.rs @@ -469,18 +469,6 @@ pub fn create_planner_accumulator( parameters: Option<&SketchParams>, input: &planner_types::summary_type::SummaryUpdate, ) -> Result, String> { - if input.item.is_some() - && matches!( - input.weight_domain, - planner_types::summary_type::WeightDomain::NonNegative { - proof: - planner_types::summary_type::NonNegativeWeightProof::ResetAwareCounterDerivative - } - ) - { - return Err("window-weighted summaries require typed DAG binding; integer heap updaters cannot consume rates".into()); - } - crate::capability::validate_summary_kernel(family, input)?; if matches!(family, SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms)) { return Err( @@ -674,128 +662,7 @@ impl AccumulatorUpdater for UnivMonUpdater { } #[cfg(test)] -mod ddsketch_domain { - use super::*; - - /// A DDSketch takes zero and negative samples (its zero bucket and - /// negative store) as well as positive ones; it refuses NaN, ±Inf, and - /// magnitudes outside its indexable range, including a nonzero magnitude - /// below it, which would lose its relative error in the zero bucket. - #[test] - fn accepts_zero_and_signed_samples_within_its_indexable_range() { - let updater = DDSketchAccumulatorUpdater::new(0.01); - let (minimum, maximum) = - asap_sketchlib::sketches::ddsketch::ddsketch_indexable_bounds(0.01); - for value in [0.0, -0.0, 3.0, -3.0, minimum, -minimum, maximum, -maximum] { - assert!(updater.validate_single_input(value).is_ok(), "{value}"); - } - for value in [ - f64::NAN, - f64::INFINITY, - f64::NEG_INFINITY, - minimum / 2.0, - -minimum / 2.0, - maximum * 2.0, - -maximum * 2.0, - ] { - assert!(updater.validate_single_input(value).is_err(), "{value}"); - } - } - - /// Quantiles over 0, 0, 3, 5 and over signed samples are within alpha of - /// PromQL's interpolated quantile, at rank q (n − 1) between adjacent - /// order statistics: here each pair of neighbours shares a sign or - /// includes zero. - #[test] - fn quantiles_over_zero_and_signed_samples() { - for (samples, expected) in [ - ( - vec![0.0, 0.0, 3.0, 5.0], - [(0.25, 0.0), (0.5, 1.5), (0.75, 3.5), (1.0, 5.0)], - ), - ( - vec![-7.0, -3.0, 0.0, 4.0], - [(0.25, -4.0), (0.5, -1.5), (0.75, 1.0), (1.0, 4.0)], - ), - ] { - let mut updater = DDSketchAccumulatorUpdater::new(0.01); - for value in &samples { - updater.validate_single_input(*value).unwrap(); - updater.update_single(*value, 0).unwrap(); - } - for (q, exact) in expected { - let estimate = updater - .acc - .estimate(&planner_types::summary_type::SketchStatistic::Quantile { q }) - .unwrap(); - assert!( - (estimate - exact).abs() <= 0.01 * f64::abs(exact), - "{samples:?} q={q}: {estimate} vs {exact}" - ); - } - } - } -} +mod ddsketch_domain; #[cfg(test)] -mod tests { - use super::*; - - /// A DDSketch updater reports sketchlib's count of the bins its stores - /// allocate: none before a sample, one 128-bin chunk of the positive - /// store after a few positive samples. - #[test] - fn ddsketch_memory_is_its_allocated_bins() { - let mut updater = DDSketchAccumulatorUpdater::new(0.05); - let empty = std::mem::size_of::(); - assert_eq!(updater.memory_usage_bytes(), empty); - for value in [1.0, 1.5, 2.0, 4.0] { - updater.update_single(value, 0).unwrap(); - } - assert_eq!(updater.memory_usage_bytes(), empty + 128 * 8); - assert_eq!( - updater.memory_usage_bytes(), - updater.acc.inner.memory_bytes() - ); - } - - /// A numeric sketch takes an Int64 as its exact Float64, and refuses - /// one beyond 2^53, which no Float64 holds exactly. - #[test] - fn a_numeric_sketch_takes_exact_int64_values() { - use crate::values::Value; - let mut updater = KllAccumulatorUpdater::new(200); - for value in 1..=5i64 { - updater.update_value(&Value::Int64(value), 0).unwrap(); - } - assert!(updater - .update_value(&Value::Int64((1 << 53) + 1), 0) - .is_err()); - assert!(updater.update_value(&Value::Utf8("x".into()), 0).is_err()); - let median = updater - .take_accumulator() - .estimate(&planner_types::summary_type::SketchStatistic::Quantile { q: 0.5 }) - .unwrap(); - assert_eq!(median, 3.0); - } - - // An HLL build takes Int64 ids and Utf8 names as items. - #[test] - fn an_hll_takes_int64_and_utf8_items() { - use crate::values::Value; - let mut updater = HllUpdater { - acc: HllSketchAccumulator::new(12).unwrap(), - }; - for id in 0..100i64 { - updater.update_value(&Value::Int64(id % 50), 0).unwrap(); - updater - .update_value(&Value::Utf8(format!("n{}", id % 10).into()), 0) - .unwrap(); - } - let estimate = updater - .take_accumulator() - .estimate(&planner_types::summary_type::SketchStatistic::Cardinality) - .unwrap(); - assert_eq!(estimate, 60.0); - } -} +mod tests; diff --git a/crates/executor/src/summary_kernels/factory/tests.rs b/crates/executor/src/summary_kernels/factory/tests.rs new file mode 100644 index 000000000..cde466f27 --- /dev/null +++ b/crates/executor/src/summary_kernels/factory/tests.rs @@ -0,0 +1,59 @@ +use super::*; + +/// A DDSketch updater reports sketchlib's count of the bins its stores +/// allocate: none before a sample, one 128-bin chunk of the positive +/// store after a few positive samples. +#[test] +fn ddsketch_memory_is_its_allocated_bins() { + let mut updater = DDSketchAccumulatorUpdater::new(0.05); + let empty = std::mem::size_of::(); + assert_eq!(updater.memory_usage_bytes(), empty); + for value in [1.0, 1.5, 2.0, 4.0] { + updater.update_single(value, 0).unwrap(); + } + assert_eq!(updater.memory_usage_bytes(), empty + 128 * 8); + assert_eq!( + updater.memory_usage_bytes(), + updater.acc.inner.memory_bytes() + ); +} + +/// A numeric sketch takes an Int64 as its exact Float64, and refuses +/// one beyond 2^53, which no Float64 holds exactly. +#[test] +fn a_numeric_sketch_takes_exact_int64_values() { + use crate::values::Value; + let mut updater = KllAccumulatorUpdater::new(200); + for value in 1..=5i64 { + updater.update_value(&Value::Int64(value), 0).unwrap(); + } + assert!(updater + .update_value(&Value::Int64((1 << 53) + 1), 0) + .is_err()); + assert!(updater.update_value(&Value::Utf8("x".into()), 0).is_err()); + let median = updater + .take_accumulator() + .estimate(&planner_types::summary_type::SketchStatistic::Quantile { q: 0.5 }) + .unwrap(); + assert_eq!(median, 3.0); +} + +// An HLL build takes Int64 ids and Utf8 names as items. +#[test] +fn an_hll_takes_int64_and_utf8_items() { + use crate::values::Value; + let mut updater = HllUpdater { + acc: HllSketchAccumulator::new(12).unwrap(), + }; + for id in 0..100i64 { + updater.update_value(&Value::Int64(id % 50), 0).unwrap(); + updater + .update_value(&Value::Utf8(format!("n{}", id % 10).into()), 0) + .unwrap(); + } + let estimate = updater + .take_accumulator() + .estimate(&planner_types::summary_type::SketchStatistic::Cardinality) + .unwrap(); + assert_eq!(estimate, 60.0); +} diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index dc7726b2b..42df794de 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -92,9 +92,6 @@ fn unit_count() -> SummaryUpdate { SummaryUpdate { item: Some(SummaryInputExpr::Column(ColumnRef::Named("service".into()))), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, } } @@ -270,26 +267,19 @@ fn hydra_cms_merges_like_one_build() { ); } -fn weighted(weight_domain: WeightDomain) -> SummaryUpdate { +fn weighted() -> SummaryUpdate { SummaryUpdate { weight: SummaryInputExpr::Column(ColumnRef::Named("weight".into())), - weight_domain, ..unit_count() } } -fn non_negative() -> WeightDomain { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - } -} - // A non-negative weight column sums each item's weight within its group // (e.g. bytes per service per job), within the Count-Min bound over the // stream's total weight; a split-stream merge answers as one build. #[test] fn weighted_hydra_cms_sums_item_weights_per_group() { - let points = run(weighted(non_negative()), None, vec![batch(&ROWS)]); + let points = run(weighted(), None, vec![batch(&ROWS)]); let total: f64 = ROWS.iter().map(|(_, _, w)| w).sum(); let slack = std::f64::consts::E * total * (1. / f64::from(SHARED_COLUMNS) + 1. / f64::from(WIDTH)); @@ -304,26 +294,19 @@ fn weighted_hydra_cms_sums_item_weights_per_group() { } let (left, right) = ROWS.split_at(5); assert_eq!( - run( - weighted(non_negative()), - None, - vec![batch(left), batch(right)] - ), + run(weighted(), None, vec![batch(left), batch(right)]), points ); } -// A positive constant weight, `Constant(2.0)` with its `PositiveConstant` -// proof, adds 2 per row: each job's point count is twice the unit count's. +// A positive constant weight, `Constant(2.0)`, adds 2 per row: each job's +// point count is twice the unit count's. // A non-integer constant is refused at binding, since sketchlib's Hydra adds // integer weights. #[test] fn positive_constant_weight_adds_the_constant_per_row() { let constant = |weight: f64| SummaryUpdate { weight: SummaryInputExpr::Constant(weight), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::PositiveConstant, - }, ..unit_count() }; let unit = run(unit_count(), None, vec![batch(&ROWS)]); @@ -347,12 +330,9 @@ fn positive_constant_weight_adds_the_constant_per_row() { fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { let constant = SummaryUpdate { weight: SummaryInputExpr::Constant(2.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::PositiveConstant, - }, ..unit_count() }; - let column = weighted(non_negative()); + let column = weighted(); for (update, expected) in [ (constant, serde_json::json!({"Constant": 2})), (column, serde_json::json!({"Column": 2})), @@ -394,9 +374,6 @@ fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { fn constant_weight_above_i32_is_refused_at_validation() { let update = SummaryUpdate { weight: SummaryInputExpr::Constant(2_147_483_648.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::PositiveConstant, - }, ..unit_count() }; let error = asap_executor::capability::validate_summary_kernel(&family(), &update) @@ -414,19 +391,6 @@ fn constant_weight_above_i32_is_refused_at_validation() { ); } -// A weight column of unknown sign is rejected at binding: a Count-Min point -// count only overestimates when no weight is negative. -#[test] -fn unknown_sign_weight_is_rejected() { - let error = compile_node( - &summary_agg(weighted(WeightDomain::UnknownOrSigned), None), - &[input_schema()], - ) - .err() - .expect("unknown-sign weight"); - assert!(error.to_string().contains("non-negative"), "{error}"); -} - // sketchlib's Hydra adds integer weights, so a negative, fractional or // non-finite weight value fails the build with a clear error. #[test] @@ -438,7 +402,7 @@ fn invalid_weight_values_fail_the_build() { (f64::INFINITY, "integer"), ] { let rows = [("api", "checkout", 1.0), ("api", "checkout", weight)]; - let error = try_run(weighted(non_negative()), None, vec![batch(&rows)]) + let error = try_run(weighted(), None, vec![batch(&rows)]) .err() .unwrap_or_else(|| panic!("{weight} accepted")); assert!(error.contains(expected), "{weight}: {error}"); diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index f262878ae..ba524ae1d 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -141,9 +141,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { ColumnRef::Named("service".into()), )])), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![0])), filter: None, diff --git a/crates/executor/tests/main.rs b/crates/executor/tests/main.rs index ee0b768e5..3aad7004d 100644 --- a/crates/executor/tests/main.rs +++ b/crates/executor/tests/main.rs @@ -28,4 +28,3 @@ mod text_extremum_memory; mod thread_safety; mod time_series_window; mod univmon_execution; -mod weighted_topk_binding; diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index a02882402..d35156b61 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -145,7 +145,6 @@ fn summary_capability_levels_are_distinct() { ColumnRef::Named("host".into()), )), weight: planner_types::summary_type::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), }; assert!(validate_summary_kernel(&cms, &update).is_ok()); // Stored Count-Min state reads only its bare count natively. diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 25d3fe675..033118a69 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -856,7 +856,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { input: SummaryUpdate { item: None, weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), filter: None, diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index fcc1ba2fd..2bfdc4135 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -17,8 +17,7 @@ use planner_types::ir::schema::{Field, FieldDataType}; use planner_types::ir::ASAPOp; use planner_types::plan::physical_asap_dag::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; use planner_types::summary_type::{ - NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, - WeightDomain, + SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; use std::sync::Arc; @@ -170,9 +169,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() input: SummaryUpdate { item: Some(SummaryInputExpr::Column(ColumnRef::Named("src_ip".into()))), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), filter: None, diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs deleted file mode 100644 index 90750507c..000000000 --- a/crates/executor/tests/weighted_topk_binding.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Planner output binds directly to the shared runtime over raw counter samples. -use planner_types::ir::schema::*; -use planner_types::summary_type::*; - -#[test] -fn rate_updates_cannot_enter_integer_heap_factory() { - let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); - let input = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("service".into()), - )), - weight: SummaryInputExpr::Column(planner_types::ir::scalar::ColumnRef::SampleValue), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - }, - }; - assert!(asap_executor::factory::create_planner_accumulator(&family, None, &input).is_err()); -} diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs index 9b8178fa4..34358eb1a 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs @@ -18,7 +18,7 @@ use asap_types::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr}; use super::pending::{add_candidate, commit_pending, Pending}; use super::SingleMeasure; use crate::computation_targets::HowToComputeAggregateMeasure; -use crate::summary_realization::{measure_input, summary_update, with_raw_data_samples_of}; +use crate::summary_realization::{measure_input, summary_update}; use crate::{schema, PerComputationCandidateError}; /// The operator that computes `measure` by `way` for @@ -53,16 +53,13 @@ pub(super) fn measure_output( measure.unfiltered_input, measure.filter, )?; - let update = with_raw_data_samples_of( - summary_update( - measure.intent, - &summary_type, - measure.reduction, - schema(memo, measure.unfiltered_input), - read, - )?, - measure.raw_data_samples_of, - ); + let update = summary_update( + measure.intent, + &summary_type, + measure.reduction, + schema(memo, measure.unfiltered_input), + read, + )?; let build = Operator::ASAP(ASAPOp::SummaryAgg { child: measure.input, input: update, diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs index a863c7551..8d95fe7d8 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/mod.rs @@ -53,13 +53,13 @@ use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr}; -use asap_types::stages::{raw_data_samples_of_metric_of_group, Memo, MemoGroupId}; +use asap_types::stages::{Memo, MemoGroupId}; use crate::computation_targets::{ nested_distributive_input, whole_expression_input, ComputationTarget, HowToComputeAggregateMeasure, }; -use crate::summary_realization::{measure_input, with_raw_data_samples_of}; +use crate::summary_realization::measure_input; use crate::{expression, PerComputationCandidateError}; use having::having_filter_groups; use measure_output::{ @@ -224,10 +224,6 @@ pub(crate) struct SingleMeasure<'a> { pub(crate) reduction: &'a Reduction, pub(crate) intent: &'a AggIntent, pub(crate) filter: Option<&'a Predicate>, - /// The metric whose raw data samples, or plain sums of them, are every - /// value the aggregate reads: a heap-sketch weight read from them - /// carries the open requirement `WeightDomain::RawDataSamplesOfMetric`. - pub(crate) raw_data_samples_of: Option<&'a str>, } /// Adds `ways.result_group`'s candidates that compute it from @@ -268,14 +264,12 @@ fn add_aggregate_candidates( } Ok((pending, input)) }; - let raw_data_samples = raw_data_samples_of_metric_of_group(memo, child); let measure = |index: usize, input: MemoGroupId| SingleMeasure { input, unfiltered_input: child, reduction: &reduction, intent: &measures[index], filter: filters.get(index).and_then(Option::as_ref), - raw_data_samples_of: raw_data_samples.as_deref(), }; let target_of = |index: usize| { ways.targets @@ -445,7 +439,7 @@ fn add_fused_candidate( )?; let build = Operator::ASAP(ASAPOp::SummaryAgg { child: rows, - input: with_raw_data_samples_of(update, measure.raw_data_samples_of), + input: update, summary_type: FieldDataType::Sketch(*algorithm), reduction: partitions, filter: None, @@ -487,7 +481,6 @@ fn add_fused_candidate( reduction: &partitions, intent: &inner, filter: None, - raw_data_samples_of: None, }; let merged = accumulator_build( memo, diff --git a/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs b/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs index 94cb6678d..1fd027824 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs @@ -11,12 +11,10 @@ use asap_types::ir::scalar::column_resolution::{resolve_column_ref, ResolveError use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr}; -use asap_types::stages::{ - ranked_column_of_group, raw_data_samples_of_metric_of_group, Memo, MemoGroupId, -}; +use asap_types::stages::{ranked_column_of_group, Memo, MemoGroupId}; use asap_types::summary_type::{ - EntityIdentity, NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, - SummaryTypeCapability, SummaryUpdate, WeightDomain, + EntityIdentity, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, + SummaryUpdate, }; use crate::computation_targets::nested_aggregates; @@ -109,9 +107,10 @@ pub(crate) fn summarised_input( /// (`TopK(Count GROUP BY key)`, `TopK(Sum GROUP BY key)`): the group the heap /// reads and its update, one keyed heap over the raw rows rather than over /// an independently materialized per-item result. The update serves both -/// heap algorithms; whether Count-Min is admissible over its weights is plan -/// selection's question. `None` when the rule does not apply or the shape is -/// unsupported (each reason is noted where it returns). +/// heap algorithms; whether Count-Min is admissible over its weights is +/// Stage 2's question (the state group's `column_signs`). `None` when the +/// rule does not apply or the shape is unsupported (each reason is noted +/// where it returns). pub(crate) fn realize_keyed_additive_summary_input( memo: &Memo, output_reduction: &Reduction, @@ -134,26 +133,17 @@ pub(crate) fn realize_keyed_additive_summary_input( if any_measure_filtered(filters) { return None; } - // A grouped sum of per-series `rate`/`increase`/`irate` values: the - // metric they derive from, whose declared type decides their sign. - let derivative_of = match (measures.as_slice(), expression(memo, raw_child)) { + // A grouped sum of per-series `rate`/`increase`/`irate` values. + let counter_input = matches!( + (measures.as_slice(), expression(memo, raw_child)), ( [AggIntent::Sum { .. }], - Operator::NonASAP(NonASAPOp::Aggregate { - measures: derivative, - child: samples, - .. - }), + Operator::NonASAP(NonASAPOp::Aggregate { measures: derivative, .. }), ) if matches!( derivative.as_slice(), [AggIntent::Rate | AggIntent::Increase | AggIntent::IRate] - ) => - { - Some(raw_data_samples_of_metric_of_group(memo, *samples)) - } - _ => None, - }; - let counter_input = derivative_of.is_some(); + ) + ); // `sum by (G) (sum_over_time(m[W]))` or `... count_over_time ...`: the // grouped sum of per-series sums (counts) of raw data samples is their sum // (count) by `G`, so the heap reads those samples. @@ -184,13 +174,6 @@ pub(crate) fn realize_keyed_additive_summary_input( }), _ => return None, }; - let weight_domain = match derivative_of { - _ if counted => WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - Some(Some(metric)) => WeightDomain::CounterDerivativeOfMetric { metric }, - _ => WeightDomain::UnknownOrSigned, - }; let subpopulation_columns = match output_reduction { Reduction::PerEntity => vec![], Reduction::Reduce(keys) => keys @@ -249,7 +232,6 @@ pub(crate) fn realize_keyed_additive_summary_input( SummaryUpdate { item: Some(item), weight, - weight_domain, }, )) } @@ -281,23 +263,6 @@ fn summed_per_series_samples( Some((nested.rows, nested.inner_measure.clone())) } -/// `update` with the open requirement that its weights are samples of -/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap -/// update whose weights are not otherwise proven. -pub(crate) fn with_raw_data_samples_of( - mut update: SummaryUpdate, - metric: Option<&str>, -) -> SummaryUpdate { - if let Some(metric) = metric { - if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { - update.weight_domain = WeightDomain::RawDataSamplesOfMetric { - metric: metric.to_string(), - }; - } - } - update -} - /// What a measure's state reads from each row of `input`, resolvable there. /// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count /// included, reads its input column (the sample value when it names none). @@ -449,23 +414,14 @@ pub(crate) fn summary_update( item: Some(item), // The ranked value ([`measure_input`]). weight, - // Not proven non-negative; selection decides whether CMS is admissible. - weight_domain: WeightDomain::UnknownOrSigned, } } (_, Some(SketchAlgorithm::UnivMon)) | (AggIntent::Count, Some(SketchAlgorithm::Cms)) => { SummaryUpdate { item: Some(weight), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, } } - _ => SummaryUpdate { - item: None, - weight, - weight_domain: WeightDomain::UnknownOrSigned, - }, + _ => SummaryUpdate { item: None, weight }, }) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs index f08d482ee..ac0dc7033 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/fused.rs @@ -6,14 +6,14 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::{ASAPOp, Operator, TimeRangeKind}; use asap_types::stages::Memo; -use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryInputExpr, WeightDomain}; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryInputExpr}; use asap_types::test_support::memo_builders::{candidates_of, query}; use asap_types::workload::DataWorkload; /// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the top-k gets heaps /// over the range's raw data samples, keyed by `job` (the ranked item), -/// weighted by the sample value with the open requirement that the samples -/// of `m` are non-negative (`RawDataSamplesOfMetric`). Each heap build +/// weighted by the sample value (whether those weights are non-negative is +/// the state group's `column_signs`, which Stage 2 reads). Each heap build /// lists the absorbed sums' groups in `fused_groups`, and the top-k's own /// heaps over its one-row-per-item input are pruned. #[test] @@ -50,10 +50,6 @@ fn whole_expression_topk_heaps_read_the_raw_samples() { update.item, Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) ); - assert_eq!( - update.weight_domain, - WeightDomain::RawDataSamplesOfMetric { metric: "m".into() } - ); for build in candidates_of(&memo, state) { let mut fused = build.fused_groups.clone(); fused.sort(); @@ -133,8 +129,7 @@ fn whole_expression_topk_heap_candidates() { assert_eq!(memo.groups().len(), 11, "{}", memo_printout(&memo)); } -/// A top-k over a `count` ranks the counted items with unit weight, proven -/// non-negative. +/// A top-k over a `count` ranks the counted items with unit weight. #[test] fn count_ranked_topk_counts_with_unit_weight() { let mut memo = Memo::new(&DataWorkload::default()); @@ -157,10 +152,6 @@ fn count_ranked_topk_counts_with_unit_weight() { let memo = pass1_memo(memo, None).unwrap(); let (_, update) = state_group(&memo, SketchAlgorithm::CmsWithHeap); assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - assert!(matches!( - update.weight_domain, - WeightDomain::NonNegative { .. } - )); } /// The ranked item over several group keys is their tuple; the top-k's own diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs b/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs index 3df8a6c01..c6ace31f6 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/tests/unsupported_ways.rs @@ -58,7 +58,6 @@ fn a_heap_estimate_without_the_groups_columns_adds_nothing() { reduction: &reduction, intent: &intent, filter: None, - raw_data_samples_of: None, }; assert_unsupported_way_adds_nothing( &mut memo, @@ -100,7 +99,6 @@ fn a_readout_an_open_group_cannot_rename_adds_nothing() { reduction: &Reduction::PerEntity, intent: &intent, filter: None, - raw_data_samples_of: None, }; assert_unsupported_way_adds_nothing( &mut memo, diff --git a/crates/stage2-physical-optimization/Cargo.toml b/crates/stage2-physical-optimization/Cargo.toml index db1f94888..26a9e74a7 100644 --- a/crates/stage2-physical-optimization/Cargo.toml +++ b/crates/stage2-physical-optimization/Cargo.toml @@ -3,10 +3,12 @@ name = "asap-stage2-physical-optimization" version = "0.1.0" edition = "2021" -# #509 Stage 2, physical ASAP-aware optimization: materialization and -# physical operator implementation of each Stage 1 candidate, and the -# PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`). Depends only on -# asap-types, never on Stage 1's crate, plan selection or the executor. +# Stage 2, physical ASAP-aware optimization: each Stage 1 candidate's +# timing and lifetime, and the PhysicalASAPMemoCandidates Stage 3 selects +# from. Depends only on asap-types, never on Stage 1's crate, plan selection +# or the executor. [dependencies] asap-types = { path = "../types" } -thiserror = "2" + +[dev-dependencies] +asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/stage2-physical-optimization/src/candidate.rs b/crates/stage2-physical-optimization/src/candidate.rs new file mode 100644 index 000000000..effe6d523 --- /dev/null +++ b/crates/stage2-physical-optimization/src/candidate.rs @@ -0,0 +1,202 @@ +//! Decision-tree levels 3 and 4 (concepts doc, section 2): each logical +//! candidate's physical properties $p^{\mathrm{phys}} = (\tau, \ell)$, the +//! product of the values each dimension offers the candidate less the +//! invalid combination (`PhysicalProperties::new` rejects it), each with +//! the reason a prune rule removes it. Each prune rule reads only the value +//! it removes and the values decided above it. + +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::operator::arguments::Source; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroupId, Lifetime, Memo, MemoGroupId, PhysicalProperties, PruneReason, + StorageMedium, ValueSign, +}; +use asap_types::summary_type::{ExactKind, SketchAlgorithm}; +use asap_types::workload::{DataArrival, DataWorkload}; + +/// Level 3's values, $\tau$. +const TIMINGS: [ExecutionTiming; 2] = [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime]; + +/// Level 4's values, $\ell$. A materialized result is stored in memory +/// only: the deployment capabilities offer no disk storage +/// (`DeploymentCapabilities` has no disk field). +const LIFETIMES: [Lifetime; 3] = [ + Lifetime::None, + Lifetime::OneEvaluation(StorageMedium::Memory), + Lifetime::AcrossEvaluations(StorageMedium::Memory), +]; + +/// One $(\tau, \ell)$ offered to a logical candidate, with the reason a +/// prune rule removes it. +pub(super) struct OfferedPhysicalProperties { + pub(super) physical_properties: PhysicalProperties, + /// `None` when no prune rule removes it. + pub(super) prune_reason: Option, +} + +/// Every valid $(\tau, \ell)$ offered to `candidate`, in [`TIMINGS`] × +/// [`LIFETIMES`] order, each with the reason a prune rule removes it. +/// Offered are: +/// +/// - `IngestionTime` only when `ingestion_time_offered_for_group` (the +/// candidate's group is time-independent or an input of one) and, for a +/// counter state, when its rows arrive in timestamp order or within a +/// known bound ([`counter_state_rows_ordered`]); +/// - `AcrossEvaluations` only when `across_evaluations_offered_for_group` +/// (a chain, a group on a known grid or a time-independent group, not +/// read under an absolute `@ T`); +/// - (IngestionTime, None) not for a summary build (`SummaryAgg`, a chain, +/// `MaintainPopulation`): fed at ingestion time, it keeps its state +/// between batches, so it is materialized. +/// +/// A value not offered is not returned, and neither is the invalid +/// (IngestionTime, OneEvaluation). +pub(super) fn offered_physical_properties( + memo: &Memo, + candidate: CandidateForMemoGroupId, + ingestion_time_offered_for_group: bool, + across_evaluations_offered_for_group: bool, + data: &DataWorkload, + capabilities: &DeploymentCapabilities, +) -> Vec { + let operator = &memo.candidate(candidate).expression.operator; + let build = summary_build(operator); + let ingestion_time_offered = ingestion_time_offered_for_group + && build.is_none_or(|(summary_type, child)| { + counter_state_rows_ordered(memo, summary_type, child, data) + }); + let count_min_weights_not_proven_non_negative = build.is_some_and(|(summary_type, _)| { + count_min_weights_not_proven_non_negative(memo, candidate.group, summary_type) + }); + let has_ingestion_pipeline = data.arrival.rows_per_second().is_some(); + let scans_raw_data = matches!(operator, Operator::NonASAP(NonASAPOp::Scan { .. })); + let builds_summary = + build.is_some() || matches!(operator, Operator::ASAP(ASAPOp::MaintainPopulation { .. })); + let mut offered = Vec::new(); + for timing in TIMINGS { + if timing == ExecutionTiming::IngestionTime && !ingestion_time_offered { + continue; + } + for lifetime in LIFETIMES { + let across_evaluations = matches!(lifetime, Lifetime::AcrossEvaluations(_)); + if across_evaluations && !across_evaluations_offered_for_group { + continue; + } + if builds_summary + && timing == ExecutionTiming::IngestionTime + && lifetime == Lifetime::None + { + continue; + } + let Ok(physical_properties) = PhysicalProperties::new(timing, lifetime) else { + continue; + }; + let query_time = timing == ExecutionTiming::QueryTime; + let prune_reason = if count_min_weights_not_proven_non_negative { + Some(PruneReason::CountMinWeightsNotProvenNonNegative) + } else if !query_time && !has_ingestion_pipeline { + Some(PruneReason::NoIngestionPipeline) + } else if (query_time || across_evaluations) + && scans_raw_data + && !capabilities.raw_data_retained + { + Some(PruneReason::RawDataNotRetained) + } else if query_time && across_evaluations && !capabilities.query_time_results_retained + { + Some(PruneReason::QueryTimeResultsNotRetained) + } else { + None + }; + offered.push(OfferedPhysicalProperties { + physical_properties, + prune_reason, + }); + } + } + offered +} + +/// The summary type and input group of `operator` when it builds a +/// summary state from rows (`SummaryAgg`, `SummaryTumblingWindowChain`). +fn summary_build(operator: &Operator) -> Option<(&FieldDataType, MemoGroupId)> { + match operator { + Operator::ASAP( + ASAPOp::SummaryAgg { + child, + summary_type, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + child, + summary_type, + .. + }, + ) => Some((summary_type, *child)), + _ => None, + } +} + +/// Whether a build of `summary_type` over group `child`, when it is a +/// counter state (`rate`, `increase`, `irate`), gets its rows in timestamp +/// order at ingestion time: the data declares how late a row may arrive +/// (`max_out_of_order_ms`), so a reorder buffer restores the order, or its +/// rows are PromQL samples, taken to arrive in order. True for any other +/// summary type. +fn counter_state_rows_ordered( + memo: &Memo, + summary_type: &FieldDataType, + child: MemoGroupId, + data: &DataWorkload, +) -> bool { + let counter_state = matches!( + summary_type, + FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase | ExactKind::IRate, _) + ); + let out_of_order_bound_known = matches!( + data.arrival, + DataArrival::Ingesting { + max_out_of_order_ms: Some(_), + .. + } | DataArrival::Mixed { + max_out_of_order_ms: Some(_), + .. + } + ); + !counter_state + || out_of_order_bound_known + || memo.groups()[child.0 as usize] + .group_properties + .scanned_sources + .iter() + .all(Source::is_time_series) +} + +/// Whether a build of `summary_type` in group `group` is a Count-Min state +/// (`Cms`, `CmsWithHeap`, `HydraCms`) whose weights are not proven +/// non-negative: the state column of its group, which records the sign of +/// the weights it is updated with (`column_signs`), is not `NonNegative`. +fn count_min_weights_not_proven_non_negative( + memo: &Memo, + group: MemoGroupId, + summary_type: &FieldDataType, +) -> bool { + if !matches!( + summary_type, + FieldDataType::Sketch( + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::HydraCms + ) + ) { + return false; + } + let properties = &memo.groups()[group.0 as usize].group_properties; + let weight_sign = properties + .schema + .fields + .iter() + .position(|field| !field.is_plain()) + .map(|state| properties.column_signs[state]); + weight_sign != Some(ValueSign::NonNegative) +} diff --git a/crates/stage2-physical-optimization/src/edge.rs b/crates/stage2-physical-optimization/src/edge.rs new file mode 100644 index 000000000..b896df397 --- /dev/null +++ b/crates/stage2-physical-optimization/src/edge.rs @@ -0,0 +1,73 @@ +//! Each logical edge expanded into the physical edges between the two +//! candidates' alternatives, with $\alpha^{\log}$ copied and +//! $\alpha^{\mathrm{phys}}$ from the producer's $(\tau, \ell)$ and the +//! consumer's $\tau$. + +use std::collections::HashMap; +use std::ops::Range; + +use asap_types::stages::{ + producer_satisfies_physical_requirements, CandidateForMemoGroupId, LogicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, +}; + +/// Every physical edge between `alternatives`, over provisional IDs +/// (indices into `alternatives`), and, per alternative and input, the +/// alternatives on admissible edges into it, which pruning reads. +/// `alternatives_of_logical_candidate` gives each logical candidate's +/// alternatives as a range of `alternatives`. Pass 2's edges are grouped by +/// (consumer, input), and those into one (c, i) are exactly the candidates +/// of $G_{c,i}$ that survived Pass 2. +pub(super) fn expand_edges( + logical: &LogicalASAPMemoCandidates, + alternatives: &[PhysicalCandidate], + alternatives_of_logical_candidate: &HashMap>, +) -> (Vec, Vec>>) { + let mut edges = Vec::new(); + let mut admissible_producer_alternatives_per_input: Vec>> = alternatives + .iter() + .map(|alternative| { + let input_count = logical + .memo + .candidate(alternative.logical_candidate_id) + .expression + .operator + .children() + .len(); + vec![Vec::new(); input_count] + }) + .collect(); + for edges_into_input in logical.edges.chunk_by(|a, b| { + (a.consumer_candidate_id, a.input_index_of_consumer) + == (b.consumer_candidate_id, b.input_index_of_consumer) + }) { + let consumer = edges_into_input[0].consumer_candidate_id; + let input_index_of_consumer = edges_into_input[0].input_index_of_consumer; + for consumer_index in alternatives_of_logical_candidate[&consumer].clone() { + let consumer_timing = alternatives[consumer_index].physical_properties.timing(); + for logical_edge in edges_into_input { + for producer_index in + alternatives_of_logical_candidate[&logical_edge.producer_candidate_id].clone() + { + let edge = PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(producer_index), + consumer_candidate_id: PhysicalCandidateId(consumer_index), + input_index_of_consumer, + satisfies_logical_requirement: logical_edge.satisfies_logical_requirement, + satisfies_physical_requirement: producer_satisfies_physical_requirements( + &alternatives[producer_index], + consumer_timing, + ), + }; + if edge.is_logically_and_physically_admissible() { + admissible_producer_alternatives_per_input[consumer_index] + [input_index_of_consumer] + .push(producer_index); + } + edges.push(edge); + } + } + } + } + (edges, admissible_producer_alternatives_per_input) +} diff --git a/crates/stage2-physical-optimization/src/group.rs b/crates/stage2-physical-optimization/src/group.rs new file mode 100644 index 000000000..ae70cb6b1 --- /dev/null +++ b/crates/stage2-physical-optimization/src/group.rs @@ -0,0 +1,142 @@ +//! Per-group facts the decision tree's levels 3 and 4 read: whether a +//! group's candidates are offered `IngestionTime`, and whether they are +//! offered `AcrossEvaluations`. Each is computed once per group, over the +//! candidates Pass 2 kept (an eliminated candidate reads nothing). + +use std::collections::HashSet; + +use asap_types::ir::operator::arguments::{AtModifier, TimeShift}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, MemoGroupId, +}; + +/// The candidates of group `group` that Pass 2 kept. +fn kept_candidates<'a>( + logical: &'a LogicalASAPMemoCandidates, + eliminated: &'a HashSet, + group: usize, +) -> impl Iterator + 'a { + logical.memo.groups()[group] + .candidates + .iter() + .enumerate() + .filter(move |(index_in_group, _)| { + !eliminated.contains(&CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group: *index_in_group, + }) + }) + .map(|(_, candidate)| candidate) +} + +/// Per group, by `MemoGroupId`: whether level 3 offers its candidates +/// `IngestionTime`. It does when the group is time-independent (it does +/// not depend on the evaluation time, as a chain does not), or when it is +/// an input, directly or through other offered groups, of a candidate of +/// an offered group: an ingestion-time candidate reads only ingestion-time +/// producers. +pub(super) fn ingestion_time_offered_per_group( + logical: &LogicalASAPMemoCandidates, + eliminated: &HashSet, +) -> Vec { + let groups = logical.memo.groups(); + let mut offered: Vec = groups + .iter() + .map(|group| !group.group_properties.depends_on_evaluation_time) + .collect(); + let mut pending: Vec = (0..groups.len()).filter(|&group| offered[group]).collect(); + while let Some(group) = pending.pop() { + for candidate in kept_candidates(logical, eliminated, group) { + for input in candidate.expression.operator.children() { + let input = input.0 as usize; + if !offered[input] { + offered[input] = true; + pending.push(input); + } + } + } + } + offered +} + +/// Per group, by `MemoGroupId`: whether level 4 offers its candidates +/// `AcrossEvaluations`. It does for a chain, for a group evaluated on a +/// known grid (`evaluation_grid_of_group`) and for a time-independent +/// group (its one result is stored as is and readable at any time), unless +/// the group is read under an absolute `@ T` +/// ([`read_at_absolute_time_per_group`]). +pub(super) fn across_evaluations_offered_per_group( + logical: &LogicalASAPMemoCandidates, + eliminated: &HashSet, +) -> Vec { + let read_at_absolute_time = read_at_absolute_time_per_group(logical, eliminated); + logical + .memo + .groups() + .iter() + .zip(&logical.evaluation_grid_of_group) + .zip(read_at_absolute_time) + .map(|((group, evaluation_grid), read_at_absolute_time)| { + let chain = matches!( + group.expression, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ); + (chain + || evaluation_grid.is_some() + || !group.group_properties.depends_on_evaluation_time) + && !read_at_absolute_time + }) + .collect() +} + +/// Per group, by `MemoGroupId`: whether it is read under an absolute +/// `@ T`, that is below a `TimeShift` anchored at a timestamp, through any +/// operators except another `@` (which sets its own evaluation time). With +/// no known end of the horizon, how far back such a reader reaches (the +/// horizon end − T + range) is unbounded, so the group is not materialized +/// across evaluations (concepts doc, section 2): in +/// `max_over_time(rate(c[1m])[5m:1m] @ 90)` neither the subquery nor the +/// `rate`, its range or the scan is. A group also read on a relative path +/// counts as read under the anchor. +fn read_at_absolute_time_per_group( + logical: &LogicalASAPMemoCandidates, + eliminated: &HashSet, +) -> Vec { + let anchor = |operator: &Operator| match operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { at: Some(at), .. }, + .. + }) => Some(matches!(at, AtModifier::Timestamp(_))), + _ => None, + }; + let group_count = logical.memo.groups().len(); + let mut read = vec![false; group_count]; + let mut pending = Vec::new(); + fn mark(input: &MemoGroupId, read: &mut [bool], pending: &mut Vec) { + let input = input.0 as usize; + if !read[input] { + read[input] = true; + pending.push(input); + } + } + for group in 0..group_count { + for candidate in kept_candidates(logical, eliminated, group) { + if anchor(&candidate.expression.operator) == Some(true) { + for input in candidate.expression.operator.children() { + mark(input, &mut read, &mut pending); + } + } + } + } + while let Some(group) = pending.pop() { + for candidate in kept_candidates(logical, eliminated, group) { + if anchor(&candidate.expression.operator).is_none() { + for input in candidate.expression.operator.children() { + mark(input, &mut read, &mut pending); + } + } + } + } + read +} diff --git a/crates/stage2-physical-optimization/src/lib.rs b/crates/stage2-physical-optimization/src/lib.rs index 5a504fc47..b3cc32315 100644 --- a/crates/stage2-physical-optimization/src/lib.rs +++ b/crates/stage2-physical-optimization/src/lib.rs @@ -1,33 +1,152 @@ -//! #509 Stage 2, physical ASAP-aware optimization (concepts doc, -//! section 3, "Stage 2: physical alternatives"). +//! Stage 2, physical ASAP-aware optimization (concepts doc, section 3, +//! "Stage 2: physical alternatives"). //! //! The planner facade calls [`generate_physical_asap_memo_candidates`] on //! Pass 2's output (`asap_types::stages::LogicalASAPMemoCandidates`): each -//! logical candidate expanded into one physical candidate per valid -//! $(\tau, \ell)$, each logical edge into its physical edges with -//! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives -//! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from -//! that output and builds the plan. The crate depends on no other stage -//! crate. +//! logical candidate expanded into one physical candidate per offered +//! $(\tau, \ell)$ (decision-tree levels 3 and 4, [`candidate`], over the +//! per-group facts of [`group`]), each logical edge into the physical edges +//! between the two candidates' alternatives with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$ ([`edge`]), and the pruned alternatives +//! ([`pruning`]; `asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 +//! selects from that output and builds the plan. The crate depends on no +//! other stage crate. + +mod candidate; +mod edge; +mod group; +mod pruning; +#[cfg(test)] +mod test_fixtures; +#[cfg(test)] +mod tests; + +use std::collections::{HashMap, HashSet}; +use std::ops::Range; use asap_types::deployment_capabilities::DeploymentCapabilities; -use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, MemoGroupId, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PrunedCandidate, +}; use asap_types::workload::DataWorkload; +use candidate::{offered_physical_properties, OfferedPhysicalProperties}; + /// Stage 2: expand every logical candidate of `logical` that survived Pass 2 /// into its physical alternatives (decision-tree levels 3–4: timing and /// lifetime) the data and the deployment's `capabilities` allow, and every /// logical edge into its physical edges with $\alpha^{\log}$ and /// $\alpha^{\mathrm{phys}}$. Every alternative rejected is recorded in the -/// output's `pruned_candidates` with its `PruneReason`; a value outside a -/// dimension's declared value set (`AcrossEvaluations` on a candidate of a -/// group with no known grid) is not offered and not recorded. Costs nothing, and -/// cannot fail. +/// output's `pruned_candidates` with its `PruneReason`: first those a +/// decision-tree prune rule removes ([`candidate`]), then those that +/// cannot be built or that no query output can read ([`pruning`]). A value +/// outside a dimension's declared value set (`AcrossEvaluations` on a +/// candidate of a time-dependent group with no known grid) is not offered +/// and not recorded. Costs nothing, and cannot fail; alignment with each +/// execution context is Stage 3's check (`PhysicalCandidate::delivers_on`). pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, data: &DataWorkload, capabilities: &DeploymentCapabilities, ) -> PhysicalASAPMemoCandidates { - let _ = (logical, data, capabilities); - todo!("#1207 PR 5: physical expansion and physical edges") + let memo = &logical.memo; + let eliminated: HashSet = logical + .pruned + .iter() + .map(|pruned| pruned.candidate) + .collect(); + let ingestion_time_offered = group::ingestion_time_offered_per_group(logical, &eliminated); + let across_evaluations_offered = + group::across_evaluations_offered_per_group(logical, &eliminated); + let mut pruned_candidates = Vec::new(); + // Levels 3 and 4: the offered alternatives, before pruning, and the + // alternatives of each logical candidate (a range of `alternatives`). + let mut alternatives = Vec::new(); + let mut alternatives_of_logical_candidate: HashMap> = + HashMap::new(); + for (group_index, group) in memo.groups().iter().enumerate() { + for index_in_group in 0..group.candidates.len() { + let logical_candidate_id = CandidateForMemoGroupId { + group: MemoGroupId(group_index as u32), + index_in_group, + }; + if eliminated.contains(&logical_candidate_id) { + continue; + } + let first = alternatives.len(); + for OfferedPhysicalProperties { + physical_properties, + prune_reason, + } in offered_physical_properties( + memo, + logical_candidate_id, + ingestion_time_offered[group_index], + across_evaluations_offered[group_index], + data, + capabilities, + ) { + let alternative = PhysicalCandidate { + logical_candidate_id, + physical_properties, + }; + match prune_reason { + Some(reason) => pruned_candidates.push(PrunedCandidate { + candidate: alternative, + reason, + }), + None => alternatives.push(alternative), + } + } + alternatives_of_logical_candidate + .insert(logical_candidate_id, first..alternatives.len()); + } + } + let (edges, admissible_producer_alternatives_per_input) = + edge::expand_edges(logical, &alternatives, &alternatives_of_logical_candidate); + // Candidates that cannot be built or that no query output reads. + let prune_reasons = pruning::prune_reasons( + &alternatives, + &admissible_producer_alternatives_per_input, + memo, + ); + let mut physical_candidate_id_of_alternative = vec![None; alternatives.len()]; + let mut physical_candidates = Vec::new(); + for ((alternative, reason), id) in alternatives + .into_iter() + .zip(prune_reasons) + .zip(&mut physical_candidate_id_of_alternative) + { + match reason { + Some(reason) => pruned_candidates.push(PrunedCandidate { + candidate: alternative, + reason, + }), + None => { + *id = Some(PhysicalCandidateId(physical_candidates.len())); + physical_candidates.push(alternative); + } + } + } + // $\mathcal E$ over $\mathcal C$: the edges between emitted candidates, + // renumbered and grouped by (consumer, input). + let mut edges: Vec = edges + .into_iter() + .filter_map(|edge| { + Some(PhysicalEdge { + producer_candidate_id: physical_candidate_id_of_alternative + [edge.producer_candidate_id.0]?, + consumer_candidate_id: physical_candidate_id_of_alternative + [edge.consumer_candidate_id.0]?, + ..edge + }) + }) + .collect(); + edges.sort_by_key(|edge| (edge.consumer_candidate_id, edge.input_index_of_consumer)); + PhysicalASAPMemoCandidates::new( + logical.clone(), + physical_candidates, + edges, + pruned_candidates, + ) } diff --git a/crates/stage2-physical-optimization/src/pruning.rs b/crates/stage2-physical-optimization/src/pruning.rs new file mode 100644 index 000000000..487df8675 --- /dev/null +++ b/crates/stage2-physical-optimization/src/pruning.rs @@ -0,0 +1,85 @@ +//! Physical alternatives no plan can use, removed before Stage 3: one with +//! an input no buildable producer can supply, or one no query output reads. +//! Both follow from the admissible edges alone; no cost is involved. + +use asap_types::ir::properties::ExecutionTiming; +use asap_types::stages::{ + producer_satisfies_physical_requirements, Memo, PhysicalCandidate, PruneReason, +}; + +/// Why each of `alternatives` cannot be used, `None` for a usable one. +/// `admissible_producer_alternatives_per_input[c][i]` lists the alternatives (by +/// index) on admissible edges into input `i` of alternative `c`. +/// +/// - [`PruneReason::InputHasNoBuildableProducer`]: not buildable, where +/// buildable is the least fixed point of "every input has a buildable +/// admissible producer". A candidate without inputs is buildable; a cycle +/// without a buildable way out is not. +/// - [`PruneReason::ReachesNoQueryOutput`]: buildable, but not reached from +/// a query output over admissible edges into buildable producers. A query +/// reads its output as a query-time consumer, so a query output is an +/// alternative of a query's root group that $\alpha^{\mathrm{phys}}$ +/// lets a query-time consumer read. +pub(super) fn prune_reasons( + alternatives: &[PhysicalCandidate], + admissible_producer_alternatives_per_input: &[Vec>], + memo: &Memo, +) -> Vec> { + let mut buildable = vec![false; alternatives.len()]; + loop { + let mut changed = false; + for (candidate, inputs) in admissible_producer_alternatives_per_input + .iter() + .enumerate() + { + if !buildable[candidate] + && inputs + .iter() + .all(|producers| producers.iter().any(|&producer| buildable[producer])) + { + buildable[candidate] = true; + changed = true; + } + } + if !changed { + break; + } + } + let root_groups = memo.root_groups(); + let mut reached = vec![false; alternatives.len()]; + let mut pending: Vec = alternatives + .iter() + .enumerate() + .filter(|(index, alternative)| { + buildable[*index] + && root_groups.contains(&alternative.logical_candidate_id.group) + && producer_satisfies_physical_requirements(alternative, ExecutionTiming::QueryTime) + }) + .map(|(index, _)| index) + .collect(); + while let Some(candidate) = pending.pop() { + if !reached[candidate] { + reached[candidate] = true; + pending.extend( + admissible_producer_alternatives_per_input[candidate] + .iter() + .flatten() + .copied() + .filter(|&producer| buildable[producer]), + ); + } + } + buildable + .into_iter() + .zip(reached) + .map(|(buildable, reached)| { + if !buildable { + Some(PruneReason::InputHasNoBuildableProducer) + } else if !reached { + Some(PruneReason::ReachesNoQueryOutput) + } else { + None + } + }) + .collect() +} diff --git a/crates/stage2-physical-optimization/src/test_fixtures.rs b/crates/stage2-physical-optimization/src/test_fixtures.rs new file mode 100644 index 000000000..8457b0bae --- /dev/null +++ b/crates/stage2-physical-optimization/src/test_fixtures.rs @@ -0,0 +1,214 @@ +//! Fixtures this crate's own tests share, beside the memo builders of +//! `asap_types::test_support` (groups with the candidates Pass 1 would add, +//! queries), so no test calls another stage crate: Pass 2's output built +//! by hand (the memo, its logical edges and grids), the running example's +//! query A, and the data workloads and deployment capabilities the tests +//! vary. +use std::collections::BTreeSet; + +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator, QueryRoot}; +use asap_types::stages::{ + CandidateForMemoGroupId, EvaluationGrid, Lifetime, LogicalASAPMemoCandidates, LogicalEdge, + Memo, MemoGroupId, MemoQuery, StorageMedium, +}; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::test_support::memo_builders::{candidate, group, scan_group}; +use asap_types::test_support::memo_fixtures::{range, readout, state}; +use asap_types::workload::{DataArrival, DataWorkload}; +use ExecutionTiming::{IngestionTime, QueryTime}; + +const MEMORY: StorageMedium = StorageMedium::Memory; +pub const QT_NONE: (ExecutionTiming, Lifetime) = (QueryTime, Lifetime::None); +pub const QT_ONE: (ExecutionTiming, Lifetime) = (QueryTime, Lifetime::OneEvaluation(MEMORY)); +pub const QT_ACROSS: (ExecutionTiming, Lifetime) = (QueryTime, Lifetime::AcrossEvaluations(MEMORY)); +pub const IT_NONE: (ExecutionTiming, Lifetime) = (IngestionTime, Lifetime::None); +pub const IT_ACROSS: (ExecutionTiming, Lifetime) = + (IngestionTime, Lifetime::AcrossEvaluations(MEMORY)); + +pub const GRID_60S: Option = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, +}); + +/// The candidate `index_in_group` of `group`. +pub fn candidate_id(group: MemoGroupId, index_in_group: usize) -> CandidateForMemoGroupId { + CandidateForMemoGroupId { + group, + index_in_group, + } +} + +/// The edge from `producer` into input `input` of `consumer`. +pub fn edge( + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input: usize, + satisfies_logical_requirement: bool, +) -> LogicalEdge { + LogicalEdge { + producer_candidate_id: producer, + consumer_candidate_id: consumer, + input_index_of_consumer: input, + satisfies_logical_requirement, + } +} + +/// The group of `operator`, with one candidate of `operator` per θ among +/// the first `count` of `algorithm`'s list (`memo_builders::sketch_state` +/// adds the whole list). +pub fn sketch_group( + memo: &mut Memo, + operator: Operator, + algorithm: SketchAlgorithm, + count: usize, +) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + for parameters in ¶meter_list(algorithm)[..count] { + candidate(memo, group, operator.clone(), Some(parameters.clone())); + } + group +} + +/// A query whose root is `root`, which may be a scalar expression +/// (`memo_builders::query` takes only a group). +pub fn add_query_root(memo: &mut Memo, root: QueryRoot) { + memo.add_query(MemoQuery { + root, + accuracy_target: None, + number_of_values_per_query_evaluation: None, + }); +} + +/// Pass 2's output over `memo` with `edges` and `evaluation_grid_of_group`: +/// nothing pruned, and no query restricted to exact candidates. +pub fn logical( + memo: Memo, + edges: Vec, + evaluation_grid_of_group: Vec>, +) -> LogicalASAPMemoCandidates { + LogicalASAPMemoCandidates { + exact_only_groups_of_query: vec![BTreeSet::new(); memo.queries().len()], + memo, + edges, + pruned: vec![], + evaluation_grid_of_group, + } +} + +/// A memo over no declared source facts. +pub fn memo() -> Memo { + Memo::new(&DataWorkload::default()) +} + +/// The running example's query A, `quantile_over_time(0.95, latency[1m])` +/// evaluated every 60 s at phase 0, after Pass 2: the scan; its last +/// minute; the KLL state group, with candidate 0 a KLL build over that +/// minute and candidate 1 a merge of the windows a `TimeRange` selects +/// from a 60 s chain over the scan; and the p95 readout, the query's root. +/// Each edge's $\alpha^{\log}$ is true. Groups read by the query evaluate +/// on its 60 s grid; the scan and the chain do not depend on the +/// evaluation time and have no grid. +pub struct Example { + pub logical: LogicalASAPMemoCandidates, + pub scan: CandidateForMemoGroupId, + pub last_minute: CandidateForMemoGroupId, + pub build: CandidateForMemoGroupId, + pub merge: CandidateForMemoGroupId, + pub chain: CandidateForMemoGroupId, + pub windows: CandidateForMemoGroupId, + pub readout: CandidateForMemoGroupId, +} + +pub fn example() -> Example { + example_read_by(QueryRoot::Operator) +} + +/// [`example`], its query's root `root` over the readout's group. +pub fn example_read_by(root: impl FnOnce(MemoGroupId) -> QueryRoot) -> Example { + let mut memo = memo(); + let scan = scan_group(&mut memo, "latency", &["job"], vec![]); + let last_minute = group(&mut memo, range(60_000, scan)); + let build_state = sketch_group( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), last_minute), + SketchAlgorithm::Kll, + 1, + ); + let chain = sketch_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }), + SketchAlgorithm::Kll, + 1, + ); + // The windows and the merge carry the chain build's θ. + let windows = sketch_group(&mut memo, range(60_000, chain), SketchAlgorithm::Kll, 1); + let merge = candidate( + &mut memo, + build_state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + Some(parameter_list(SketchAlgorithm::Kll)[0].clone()), + ); + let readout = group( + &mut memo, + readout(SketchStatistic::Quantile { q: 0.95 }, build_state), + ); + add_query_root(&mut memo, root(readout)); + let [scan, last_minute, build, chain, windows, readout] = + [scan, last_minute, build_state, chain, windows, readout] + .map(|group| candidate_id(group, 0)); + let edges = vec![ + edge(scan, last_minute, 0, true), + edge(last_minute, build, 0, true), + edge(windows, merge, 0, true), + edge(scan, chain, 0, true), + edge(chain, windows, 0, true), + edge(build, readout, 0, true), + edge(merge, readout, 0, true), + ]; + let mut evaluation_grid_of_group = vec![GRID_60S; memo.groups().len()]; + evaluation_grid_of_group[scan.group.0 as usize] = None; + evaluation_grid_of_group[chain.group.0 as usize] = None; + Example { + logical: logical(memo, edges, evaluation_grid_of_group), + scan, + last_minute, + build, + merge, + chain, + windows, + readout, + } +} + +pub fn ingesting(max_out_of_order_ms: Option) -> DataWorkload { + DataWorkload { + arrival: DataArrival::Ingesting { + rows_per_second: 1000.0, + max_out_of_order_ms, + }, + ..DataWorkload::default() + } +} + +/// Raw data and query-time results retained. +pub fn retaining_everything() -> DeploymentCapabilities { + DeploymentCapabilities { + query_time_results_retained: true, + ..DeploymentCapabilities::default() + } +} diff --git a/crates/stage2-physical-optimization/src/tests.rs b/crates/stage2-physical-optimization/src/tests.rs new file mode 100644 index 000000000..0bf0e2dc6 --- /dev/null +++ b/crates/stage2-physical-optimization/src/tests.rs @@ -0,0 +1,114 @@ +//! Stage 2's tests, one module per step. Each builds Pass 2's output, the +//! memo and its logical edges, from `crate::test_fixtures` and reads the +//! physical candidates, edges and pruned alternatives through the checkers +//! below; no other stage crate is called. +mod candidates; +mod edges; +mod pruning; + +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::stages::{ + CandidateForMemoGroupId, Lifetime, LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates, + PhysicalCandidateId, PhysicalProperties, PruneReason, +}; +use asap_types::workload::DataWorkload; + +use crate::generate_physical_asap_memo_candidates; + +/// Stage 2 over `logical`, checking what Stage 3 relies on: the edges are +/// grouped by (consumer, input) over emitted candidates, and every emitted +/// candidate has an admissible edge into each input. +fn generate( + logical: &LogicalASAPMemoCandidates, + data: &DataWorkload, + capabilities: &DeploymentCapabilities, +) -> PhysicalASAPMemoCandidates { + let physical = generate_physical_asap_memo_candidates(logical, data, capabilities); + let count = physical.physical_candidates.len(); + let mut consumer_inputs = Vec::new(); + for (_, edge) in physical.edges() { + assert!(edge.producer_candidate_id.0 < count && edge.consumer_candidate_id.0 < count); + consumer_inputs.push((edge.consumer_candidate_id, edge.input_index_of_consumer)); + } + assert!(consumer_inputs.is_sorted(), "grouped by (consumer, input)"); + for (id, candidate) in physical.physical_candidates.iter().enumerate() { + let inputs = logical + .memo + .candidate(candidate.logical_candidate_id) + .expression + .operator + .children() + .len(); + for input in 0..inputs { + assert!( + !physical + .admissible_edges_into(PhysicalCandidateId(id), input) + .is_empty(), + "{candidate:?} input {input}" + ); + } + } + physical +} + +fn properties((timing, lifetime): (ExecutionTiming, Lifetime)) -> PhysicalProperties { + PhysicalProperties::new(timing, lifetime).unwrap() +} + +/// The physical properties of `logical_candidate`'s emitted alternatives. +fn offered( + physical: &PhysicalASAPMemoCandidates, + logical_candidate: CandidateForMemoGroupId, +) -> Vec { + physical + .physical_candidates + .iter() + .filter(|candidate| candidate.logical_candidate_id == logical_candidate) + .map(|candidate| candidate.physical_properties) + .collect() +} + +/// The pruned alternatives of `logical_candidate`, with their reasons. +fn pruned( + physical: &PhysicalASAPMemoCandidates, + logical_candidate: CandidateForMemoGroupId, +) -> Vec<(PhysicalProperties, PruneReason)> { + physical + .pruned_candidates + .iter() + .filter(|pruned| pruned.candidate.logical_candidate_id == logical_candidate) + .map(|pruned| (pruned.candidate.physical_properties, pruned.reason)) + .collect() +} + +/// The physical properties of each (τ, ℓ) in `timings_and_lifetimes`. +fn physical_properties_list( + timings_and_lifetimes: &[(ExecutionTiming, Lifetime)], +) -> Vec { + timings_and_lifetimes + .iter() + .copied() + .map(properties) + .collect() +} + +/// Each (τ, ℓ) in `timings_and_lifetimes`, pruned with `reason`. +fn pruned_with( + timings_and_lifetimes: &[(ExecutionTiming, Lifetime)], + reason: PruneReason, +) -> Vec<(PhysicalProperties, PruneReason)> { + timings_and_lifetimes + .iter() + .map(|&timing_and_lifetime| (properties(timing_and_lifetime), reason)) + .collect() +} + +/// Whether any of `alternatives` is materialized across evaluations. +fn any_across_evaluations<'a>( + alternatives: impl IntoIterator, +) -> bool { + alternatives + .into_iter() + .any(|properties| matches!(properties.lifetime(), Lifetime::AcrossEvaluations(_))) +} diff --git a/crates/stage2-physical-optimization/src/tests/candidates.rs b/crates/stage2-physical-optimization/src/tests/candidates.rs new file mode 100644 index 000000000..deb1be1b7 --- /dev/null +++ b/crates/stage2-physical-optimization/src/tests/candidates.rs @@ -0,0 +1,491 @@ +//! Decision-tree levels 3 and 4: the (τ, ℓ) each logical candidate is +//! offered, and the prune rules that remove some of them. +use std::collections::BTreeMap; + +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, Source}; +use asap_types::ir::properties::ExecutionTiming::QueryTime; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{Memo, PhysicalASAPMemoCandidates, PruneReason, ValueSign}; +use asap_types::summary_type::{ + EntityIdentity, ExactKind, SketchAlgorithm, SummaryInputExpr, SummaryUpdate, +}; +use asap_types::test_support::memo_builders::{group, query, scan_group}; +use asap_types::test_support::memo_fixtures::{range, shift, state}; +use asap_types::workload::{DataArrival, DataWorkload, SourceFacts}; + +use super::{ + any_across_evaluations, generate, offered, physical_properties_list, properties, pruned, + pruned_with, +}; +use crate::test_fixtures::{ + candidate_id, edge, example, ingesting, logical, memo, retaining_everything, sketch_group, + GRID_60S, IT_ACROSS, IT_NONE, QT_ACROSS, QT_NONE, QT_ONE, +}; + +/// Query A over ingesting data, everything retained. Time-dependent +/// candidates (the last minute, the build over it, the merge, the windows, +/// the readout) run only at query time; on the 60 s grid they may be +/// materialized across evaluations. The scan and the chain are +/// time-independent, so ingestion time is offered too, and so is +/// `AcrossEvaluations` without a grid: their one result is readable at any +/// time. A summary build is not offered (IT, None): fed at ingestion time +/// it keeps its state, so the chain is offered (IT, AcrossEvaluations) +/// only; the scan's (IT, None) feeds the chain built at ingestion time. +#[test] +fn offers_each_candidate_its_timings_and_lifetimes() { + let example = example(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + let query_time = physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS]); + for candidate in [ + example.last_minute, + example.build, + example.merge, + example.windows, + example.readout, + ] { + assert_eq!(offered(&physical, candidate), query_time, "{candidate:?}"); + assert_eq!(pruned(&physical, candidate), vec![], "{candidate:?}"); + } + assert_eq!( + offered(&physical, example.scan), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS, IT_NONE, IT_ACROSS]) + ); + assert_eq!( + offered(&physical, example.chain), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS]) + ); + assert_eq!(pruned(&physical, example.chain), vec![]); +} + +/// No summary build (`SummaryAgg`, chain, +/// `MaintainPopulation`) is offered (IngestionTime, None), offered or +/// pruned; the scan, which keeps nothing between batches, is. +#[test] +fn no_summary_build_is_offered_at_ingestion_time_without_a_lifetime() { + let example = example(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + let memo = &example.logical.memo; + let ingestion_time_none = properties(IT_NONE); + let alternatives = physical + .physical_candidates + .iter() + .map(|candidate| { + ( + candidate.logical_candidate_id, + candidate.physical_properties, + ) + }) + .chain(physical.pruned_candidates.iter().map(|pruned| { + ( + pruned.candidate.logical_candidate_id, + pruned.candidate.physical_properties, + ) + })); + for (logical_candidate, physical_properties) in alternatives { + let builds = matches!( + memo.candidate(logical_candidate).expression.operator, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::MaintainPopulation { .. } + ) + ); + assert!( + !(builds && physical_properties == ingestion_time_none), + "{logical_candidate:?}" + ); + } + assert!(offered(&physical, example.scan).contains(&ingestion_time_none)); +} + +/// Data at rest (or of unknown arrival) has no ingestion pipeline: every +/// ingestion-time alternative is pruned. +#[test] +fn without_an_ingestion_pipeline_nothing_runs_at_ingestion_time() { + let example = example(); + for arrival in [DataArrival::AtRest, DataArrival::Unknown] { + let data = DataWorkload { + arrival, + ..DataWorkload::default() + }; + let physical = generate(&example.logical, &data, &retaining_everything()); + assert_eq!( + pruned(&physical, example.scan), + pruned_with(&[IT_NONE, IT_ACROSS], PruneReason::NoIngestionPipeline) + ); + assert_eq!( + pruned(&physical, example.chain), + pruned_with(&[IT_ACROSS], PruneReason::NoIngestionPipeline) + ); + assert!(physical + .physical_candidates + .iter() + .all(|candidate| candidate.physical_properties.timing() == QueryTime)); + } +} + +/// Without raw data retained, the scan cannot run at query time, nor keep +/// its raw rows across evaluations; what reads it at +/// query time then has no buildable producer, so only the ingestion-time +/// chain serves the query. +#[test] +fn without_raw_data_only_ingestion_time_builds_read_the_scan() { + let example = example(); + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..retaining_everything() + }; + let physical = generate(&example.logical, &ingesting(None), &capabilities); + assert_eq!( + pruned(&physical, example.scan), + pruned_with( + &[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS], + PruneReason::RawDataNotRetained + ) + ); + assert_eq!( + offered(&physical, example.scan), + physical_properties_list(&[IT_NONE]) + ); + assert_eq!(offered(&physical, example.last_minute), vec![]); + assert_eq!(offered(&physical, example.build), vec![]); + assert_eq!( + offered(&physical, example.chain), + physical_properties_list(&[IT_ACROSS]) + ); + assert_eq!( + offered(&physical, example.merge), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS]) + ); + assert_eq!( + pruned(&physical, example.build), + pruned_with( + &[QT_NONE, QT_ONE, QT_ACROSS], + PruneReason::InputHasNoBuildableProducer + ) + ); +} + +/// The default deployment retains no query-time result across +/// evaluations: every (QueryTime, AcrossEvaluations) is pruned, and +/// `OneEvaluation` stays. +#[test] +fn query_time_results_across_evaluations_need_the_deployment_to_retain_them() { + let example = example(); + let physical = generate( + &example.logical, + &ingesting(None), + &DeploymentCapabilities::default(), + ); + for candidate in [example.build, example.merge, example.readout] { + assert_eq!( + pruned(&physical, candidate), + pruned_with(&[QT_ACROSS], PruneReason::QueryTimeResultsNotRetained), + "{candidate:?}" + ); + assert_eq!( + offered(&physical, candidate), + physical_properties_list(&[QT_NONE, QT_ONE]) + ); + } +} + +/// `m[1m] @ 1000`: the `@ T` result is time-independent, so it and its +/// time-dependent input, the range under it, are offered ingestion time. +/// The pinned result is also offered `AcrossEvaluations` without a grid, +/// and built at ingestion time it reads the range's (IT, None); the +/// range, read under the anchor, is not materialized across evaluations. +/// A query reads its output at query time, so the pinned (IT, None) is +/// pruned. +#[test] +fn an_at_t_subtree_is_offered_ingestion_time() { + let mut memo = memo(); + let scan = scan_group(&mut memo, "m", &[], vec![]); + let last_minute = group(&mut memo, range(60_000, scan)); + let pinned = group( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_000)), last_minute), + ); + query(&mut memo, pinned, None); + let [scan, last_minute, pinned] = + [scan, last_minute, pinned].map(|group| candidate_id(group, 0)); + let evaluation_grid_of_group = vec![None; memo.groups().len()]; + let logical = logical( + memo, + vec![ + edge(scan, last_minute, 0, true), + edge(last_minute, pinned, 0, true), + ], + evaluation_grid_of_group, + ); + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + assert_eq!( + offered(&physical, last_minute), + physical_properties_list(&[QT_NONE, QT_ONE, IT_NONE]) + ); + assert_eq!(pruned(&physical, last_minute), vec![]); + assert_eq!( + offered(&physical, pinned), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS]) + ); + assert_eq!( + pruned(&physical, pinned), + pruned_with(&[IT_NONE], PruneReason::ReachesNoQueryOutput) + ); +} + +/// `x @ 1700000000 [5m]` with no known horizon +/// end reads `x` back without bound, so the scan under the `@ T` is not +/// materialized across evaluations; it still runs at query time. +#[test] +fn a_subtree_under_an_absolute_anchor_is_not_materialized_across_evaluations() { + let mut memo = memo(); + let scan = scan_group(&mut memo, "x", &[], vec![]); + let pinned = group( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_700_000_000_000)), scan), + ); + let window = group(&mut memo, range(300_000, pinned)); + query(&mut memo, window, None); + let [scan, pinned, window] = [scan, pinned, window].map(|group| candidate_id(group, 0)); + let mut evaluation_grid_of_group = vec![GRID_60S; memo.groups().len()]; + evaluation_grid_of_group[pinned.group.0 as usize] = None; + evaluation_grid_of_group[window.group.0 as usize] = None; + let logical = logical( + memo, + vec![edge(scan, pinned, 0, true), edge(pinned, window, 0, true)], + evaluation_grid_of_group, + ); + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + let scan_offered = offered(&physical, scan); + assert!(scan_offered.contains(&properties(QT_NONE))); + assert!(!any_across_evaluations(&scan_offered)); + assert!(!any_across_evaluations( + pruned(&physical, scan) + .iter() + .map(|(properties, _)| properties) + )); +} + +/// A rate state built over a SQL table (rows without a series order) is +/// offered ingestion time only when the data bounds how late a row +/// arrives; over PromQL samples, taken to arrive in order, it always is. +#[test] +fn a_counter_state_runs_at_ingestion_time_over_ordered_rows() { + // A scan's schema with its series identity realized, as for a PromQL + // scan. + let mut scratch = memo(); + let rows = scan_group(&mut scratch, "rows", &[], vec![]); + let schema = scratch.groups()[rows.0 as usize] + .group_properties + .schema + .clone(); + let rate_state = |memo: &mut Memo, source: Source| { + let scan = group( + memo, + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates: vec![], + schema: schema.clone(), + }), + ); + let state = group( + memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: FieldDataType::exact(ExactKind::Rate), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }), + ); + query(memo, state, None); + (candidate_id(scan, 0), candidate_id(state, 0)) + }; + let mut memo = memo(); + let (table, table_state) = rate_state( + &mut memo, + Source::Table { + table_ref: "events".into(), + }, + ); + let (series, series_state) = rate_state( + &mut memo, + Source::TimeSeries { + metric: "c_total".into(), + }, + ); + let evaluation_grid_of_group = vec![None; memo.groups().len()]; + let logical = logical( + memo, + vec![ + edge(table, table_state, 0, true), + edge(series, series_state, 0, true), + ], + evaluation_grid_of_group, + ); + let at_ingestion = |physical: &PhysicalASAPMemoCandidates, state| { + offered(physical, state).contains(&properties(IT_ACROSS)) + }; + let unbounded = generate(&logical, &ingesting(None), &retaining_everything()); + assert!(!at_ingestion(&unbounded, table_state)); + assert!(at_ingestion(&unbounded, series_state)); + let bounded = generate(&logical, &ingesting(Some(2_000)), &retaining_everything()); + assert!(at_ingestion(&bounded, table_state)); + assert!(at_ingestion(&bounded, series_state)); +} + +/// `topk(5, count by (job) (m))` as a Count-Min heap keyed by `job` and +/// weighted by the sample value: pruned in every alternative unless the +/// data declares `m`'s values non-negative, and then offered. +#[test] +fn a_count_min_build_needs_weights_proven_non_negative() { + let heap = |column_signs: BTreeMap| { + let data = DataWorkload { + sources: [( + "m".to_string(), + SourceFacts { + column_signs, + ..SourceFacts::default() + }, + )] + .into(), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + let scan = scan_group(&mut memo, "m", &["job"], vec![]); + let state = sketch_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap), + input: SummaryUpdate { + item: Some(SummaryInputExpr::EntityIdentity( + EntityIdentity::PromqlLabelSet { excluding: vec![] }, + )), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + }, + reduction: Reduction::Reduce(Default::default()), + filter: None, + }), + SketchAlgorithm::CmsWithHeap, + 1, + ); + query(&mut memo, state, None); + let (scan, state) = (candidate_id(scan, 0), candidate_id(state, 0)); + let evaluation_grid_of_group = vec![None; memo.groups().len()]; + let logical = logical( + memo, + vec![edge(scan, state, 0, true)], + evaluation_grid_of_group, + ); + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + (offered(&physical, state), pruned(&physical, state)) + }; + let (kept, unproven) = heap(BTreeMap::new()); + assert_eq!(kept, vec![]); + assert_eq!( + unproven, + pruned_with( + &[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS], + PruneReason::CountMinWeightsNotProvenNonNegative + ) + ); + let (kept, unproven) = heap(BTreeMap::from([( + "value".to_string(), + ValueSign::NonNegative, + )])); + assert_eq!( + kept, + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS]) + ); + assert_eq!(unproven, vec![]); +} + +/// `(x[1m]) @ T` lowers as `TimeShift@T(TimeRange 1m (Scan))`: the scan is +/// read back without bound through the range just as the range is, so +/// neither is materialized across evaluations, offered or pruned. +#[test] +fn every_group_below_an_absolute_anchor_is_not_materialized_across_evaluations() { + let mut memo = memo(); + let scan = scan_group(&mut memo, "x", &[], vec![]); + let last_minute = group(&mut memo, range(60_000, scan)); + let pinned = group( + &mut memo, + shift( + 0, + Some(AtModifier::Timestamp(1_700_000_000_000)), + last_minute, + ), + ); + query(&mut memo, pinned, None); + let [scan, last_minute, pinned] = + [scan, last_minute, pinned].map(|group| candidate_id(group, 0)); + let evaluation_grid_of_group = vec![GRID_60S; memo.groups().len()]; + let logical = logical( + memo, + vec![ + edge(scan, last_minute, 0, true), + edge(last_minute, pinned, 0, true), + ], + evaluation_grid_of_group, + ); + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + for candidate in [scan, last_minute] { + let alternatives = offered(&physical, candidate); + let pruned_alternatives = pruned(&physical, candidate); + assert!( + !any_across_evaluations( + alternatives + .iter() + .chain(pruned_alternatives.iter().map(|(properties, _)| properties)) + ), + "{candidate:?}: {alternatives:?}" + ); + } + assert!(offered(&physical, scan).contains(&properties(QT_NONE))); +} + +/// `max_over_time(m[1m] @ 90)` on a fixed interval without a phase (no +/// known grid): the `Max` state over the pinned minute does not depend on +/// the evaluation time, so its one result is materialized across +/// evaluations and read at any time; built at ingestion time, it reads +/// the pinned minute's ingestion-time alternative, which is then kept. +#[test] +fn a_time_independent_group_without_a_grid_is_materialized_across_evaluations() { + let mut memo = memo(); + let scan = scan_group(&mut memo, "m", &[], vec![]); + let last_minute = group(&mut memo, range(60_000, scan)); + let pinned = group( + &mut memo, + shift(0, Some(AtModifier::Timestamp(90_000)), last_minute), + ); + let max = group( + &mut memo, + state(FieldDataType::exact(ExactKind::Max), pinned), + ); + query(&mut memo, max, None); + let [scan, last_minute, pinned, max] = + [scan, last_minute, pinned, max].map(|group| candidate_id(group, 0)); + let evaluation_grid_of_group = vec![None; memo.groups().len()]; + let logical = logical( + memo, + vec![ + edge(scan, last_minute, 0, true), + edge(last_minute, pinned, 0, true), + edge(pinned, max, 0, true), + ], + evaluation_grid_of_group, + ); + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + assert_eq!( + offered(&physical, max), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS]) + ); + assert!(offered(&physical, pinned).contains(&properties(IT_NONE))); + assert_eq!(pruned(&physical, pinned), vec![]); +} diff --git a/crates/stage2-physical-optimization/src/tests/edges.rs b/crates/stage2-physical-optimization/src/tests/edges.rs new file mode 100644 index 000000000..ec0003c1d --- /dev/null +++ b/crates/stage2-physical-optimization/src/tests/edges.rs @@ -0,0 +1,39 @@ +//! Physical edges: each logical edge expanded between the two candidates' +//! alternatives. +use asap_types::ir::properties::ExecutionTiming::IngestionTime; +use asap_types::stages::{Lifetime, PhysicalCandidateId}; + +use super::generate; +use crate::test_fixtures::{example, ingesting, retaining_everything}; + +/// Each logical edge becomes an edge between every pair of the two +/// candidates' emitted alternatives, with $\alpha^{\log}$ copied and +/// $\alpha^{\mathrm{phys}}$ from the producer's timing and lifetime and the +/// consumer's timing: the chain's four alternatives read the scan's five +/// (20 edges); a query-time chain reads the query-time scan and the scan +/// kept across evaluations at ingestion time, and the ingestion-time chain +/// only the ingestion-time scan. +#[test] +fn edges_carry_both_requirements() { + let example = example(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + let candidate = |id: PhysicalCandidateId| physical.physical_candidates[id.0]; + let into_chain: Vec<_> = physical + .edges() + .map(|(_, edge)| edge) + .filter(|edge| candidate(edge.consumer_candidate_id).logical_candidate_id == example.chain) + .collect(); + assert_eq!(into_chain.len(), 20); + for edge in into_chain { + let producer = candidate(edge.producer_candidate_id).physical_properties; + let consumer = candidate(edge.consumer_candidate_id).physical_properties; + assert!(edge.satisfies_logical_requirement); + assert_eq!( + edge.satisfies_physical_requirement, + producer.timing() == consumer.timing() + || (producer.timing() == IngestionTime + && matches!(producer.lifetime(), Lifetime::AcrossEvaluations(_))), + "{producer:?} -> {consumer:?}" + ); + } +} diff --git a/crates/stage2-physical-optimization/src/tests/pruning.rs b/crates/stage2-physical-optimization/src/tests/pruning.rs new file mode 100644 index 000000000..724c74f91 --- /dev/null +++ b/crates/stage2-physical-optimization/src/tests/pruning.rs @@ -0,0 +1,90 @@ +//! Pruning after expansion: alternatives that cannot be built or that no +//! query output reads, and logical candidates Pass 2 eliminated. +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::ScalarValue; +use asap_types::ir::{ExprSemantics, QueryRoot, ScalarExpr}; +use asap_types::stages::{PruneCause, PruneReason, PrunedLogicalCandidate, RootPruneReason}; + +use super::{generate, offered, physical_properties_list, pruned, pruned_with}; +use crate::test_fixtures::{ + example, example_read_by, ingesting, retaining_everything, QT_ACROSS, QT_NONE, QT_ONE, +}; + +/// A logical edge with $\alpha^{\log}$ false is kept but not admissible: +/// a readout whose only producer fails it has no buildable producer, and +/// what only it reads reaches no query output. +#[test] +fn an_input_without_an_admissible_producer_is_not_buildable() { + let mut example = example(); + for logical_edge in &mut example.logical.edges { + if logical_edge.consumer_candidate_id == example.readout { + logical_edge.satisfies_logical_requirement = false; + } + } + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + assert!(physical.physical_candidates.is_empty()); + assert_eq!( + pruned(&physical, example.readout), + pruned_with( + &[QT_NONE, QT_ONE, QT_ACROSS], + PruneReason::InputHasNoBuildableProducer + ) + ); + assert_eq!( + pruned(&physical, example.build), + pruned_with( + &[QT_NONE, QT_ONE, QT_ACROSS], + PruneReason::ReachesNoQueryOutput + ) + ); +} + +/// A candidate Pass 2 eliminated is not expanded: the build over the last +/// minute, here eliminated with its input, leaves the merge as the +/// readout's only producer. +#[test] +fn candidates_pass_2_eliminated_are_not_expanded() { + let mut example = example(); + let (build, last_minute) = (example.build, example.last_minute); + example.logical.edges.retain(|edge| { + ![edge.producer_candidate_id, edge.consumer_candidate_id].contains(&build) + && edge.consumer_candidate_id != last_minute + }); + example.logical.pruned = [build, last_minute] + .map(|candidate| PrunedLogicalCandidate { + candidate, + causes: vec![PruneCause { + through_pruned_candidates: vec![], + root: RootPruneReason::NoConsumer, + }], + }) + .to_vec(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + for candidate in [build, last_minute] { + assert_eq!(offered(&physical, candidate), vec![]); + assert_eq!(pruned(&physical, candidate), vec![]); + } + assert_eq!( + offered(&physical, example.readout), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS]) + ); +} + +/// A scalar query's operand is a query output too: `latency_p95 * 2` over +/// the readout keeps the readout's query-time alternatives. +#[test] +fn a_scalar_roots_operand_is_a_query_output() { + let example = example_read_by(|readout| { + QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::PromqlScalarFromVector(readout)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }) + }); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + assert_eq!( + offered(&physical, example.readout), + physical_properties_list(&[QT_NONE, QT_ONE, QT_ACROSS]) + ); +} diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index a6f65a4cd..77df2e67e 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -126,7 +126,7 @@ impl From> for Field { /// /// "Sample" means two things in the planner, and the names keep them apart: /// a *raw data sample* is one timestamped value of a time series as it -/// arrives (`RawDataSamplesOfMetric`), and a *summary sample* is the subset of +/// arrives (a time-series `Scan`'s rows), and a *summary sample* is the subset of /// input rows a sampling summary family keeps as its state /// ([`FieldDataType::SummarySample`], [`SamplingKind`]). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -356,65 +356,4 @@ impl Schema { // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] -mod tests { - use super::*; - - fn col(name: &str, dtype: DataType) -> Field { - Field::plain(name, dtype, false) - } - - #[test] - fn schema_new_has_no_time_or_unique_key() { - let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); - assert!(s.time_index.is_none()); - assert!(!s.has_unique_key()); - assert_eq!(s.column_id("k"), Some(0)); - assert_eq!(s.column_id("v"), Some(1)); - assert_eq!(s.column_id("missing"), None); - } - - #[test] - fn schema_with_time_index_populates_metadata() { - let s = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - assert_eq!(s.time_index, Some(0)); - assert!(s.has_unique_key()); - assert_eq!(s.unique_keys, vec![vec![0, 1]]); - } - - #[test] - fn add_unique_key_dedupes() { - let mut s = Schema::new(vec![col("a", DataType::Utf8), col("b", DataType::Utf8)]); - s.add_unique_key(vec![0]); - s.add_unique_key(vec![0]); - s.add_unique_key(vec![0, 1]); - assert_eq!(s.unique_keys, vec![vec![0], vec![0, 1]]); - } - - #[test] - fn schema_serde_roundtrip() { - let s = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("value", DataType::Float64), - ], - 0, - vec![vec![0]], - ); - let json = serde_json::to_string(&s).unwrap(); - let back: Schema = serde_json::from_str(&json).unwrap(); - assert_eq!(s, back); - } - - #[test] - fn schema_without_fields_is_rejected() { - assert!(serde_json::from_str::(r#"{"closed":true}"#).is_err()); - } -} +mod tests; diff --git a/crates/types/src/ir/schema/tests.rs b/crates/types/src/ir/schema/tests.rs new file mode 100644 index 000000000..8fd710777 --- /dev/null +++ b/crates/types/src/ir/schema/tests.rs @@ -0,0 +1,60 @@ +use super::*; + +fn col(name: &str, dtype: DataType) -> Field { + Field::plain(name, dtype, false) +} + +#[test] +fn schema_new_has_no_time_or_unique_key() { + let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); + assert!(s.time_index.is_none()); + assert!(!s.has_unique_key()); + assert_eq!(s.column_id("k"), Some(0)); + assert_eq!(s.column_id("v"), Some(1)); + assert_eq!(s.column_id("missing"), None); +} + +#[test] +fn schema_with_time_index_populates_metadata() { + let s = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + assert_eq!(s.time_index, Some(0)); + assert!(s.has_unique_key()); + assert_eq!(s.unique_keys, vec![vec![0, 1]]); +} + +#[test] +fn add_unique_key_dedupes() { + let mut s = Schema::new(vec![col("a", DataType::Utf8), col("b", DataType::Utf8)]); + s.add_unique_key(vec![0]); + s.add_unique_key(vec![0]); + s.add_unique_key(vec![0, 1]); + assert_eq!(s.unique_keys, vec![vec![0], vec![0, 1]]); +} + +#[test] +fn schema_serde_roundtrip() { + let s = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![vec![0]], + ); + let json = serde_json::to_string(&s).unwrap(); + let back: Schema = serde_json::from_str(&json).unwrap(); + assert_eq!(s, back); +} + +#[test] +fn schema_without_fields_is_rejected() { + assert!(serde_json::from_str::(r#"{"closed":true}"#).is_err()); +} diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk/mod.rs similarity index 69% rename from crates/types/src/ir/schema/topk.rs rename to crates/types/src/ir/schema/topk/mod.rs index a8c82c122..ad4cb7a8c 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk/mod.rs @@ -227,106 +227,4 @@ pub fn series_identity_item( } #[cfg(test)] -mod tests { - use std::rc::Rc; - use std::time::Duration; - - use crate::ir::operator::arguments::{Reduction, Source}; - use crate::ir::operator::AggIntent; - use crate::ir::scalar::ColumnRef; - use crate::ir::schema::series_identity::with_promql_series_identity; - use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; - use crate::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, ScalarExpr, TimeRangeKind, - }; - use crate::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate, WeightDomain}; - - fn node(operator: Operator) -> Rc { - OperatorNode::new_shared(operator).unwrap() - } - - /// `topk by (job) (10, sum_over_time(m[1m]))` over series whose identity - /// is realized: the exact top-k and a heap's ranked readout of the same - /// rows (a `Project` naming the estimate's columns as the top-k's) - /// derive the same unique keys, one row per (job, series). - #[test] - fn exact_topk_and_heap_readout_derive_the_same_unique_keys() { - let open = Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ); - let rows = with_promql_series_identity(&node(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: open, - }))) - .unwrap(); - let window = node(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: rows, - })); - let per_series = node(Operator::NonASAP(NonASAPOp::Aggregate { - child: Rc::clone(&window), - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![String::new()], - filters: vec![None], - having: None, - })); - let exact = node(Operator::NonASAP(NonASAPOp::Aggregate { - child: per_series, - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::TopK { k: 10 }], - output_names: vec![String::new()], - filters: vec![None], - having: None, - })); - let heap = node(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap), - input: SummaryUpdate { - item: Some(super::series_identity_item(&window.schema, &[2])), - weight: crate::summary_type::SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }, - reduction: Reduction::by(vec![2]), - filter: None, - child: window, - })); - let estimate = node(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: heap, - query: SketchStatistic::TopK { k: 10 }, - })); - let estimated = estimate.schema.fields.len() - 1; - let cols = exact - .schema - .fields - .iter() - .enumerate() - .map(|(index, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(if index == 1 { - estimated - } else { - estimate - .schema - .fields - .iter() - .position(|input| input.name == field.name) - .unwrap() - }), - }) - .collect(); - let readout = node(Operator::NonASAP(NonASAPOp::Project { - cols, - child: estimate, - })); - assert!(!exact.schema.unique_keys.is_empty()); - assert_eq!(exact.schema.unique_keys, readout.schema.unique_keys); - } -} +mod tests; diff --git a/crates/types/src/ir/schema/topk/tests.rs b/crates/types/src/ir/schema/topk/tests.rs new file mode 100644 index 000000000..934665946 --- /dev/null +++ b/crates/types/src/ir/schema/topk/tests.rs @@ -0,0 +1,100 @@ +use std::rc::Rc; +use std::time::Duration; + +use crate::ir::operator::arguments::{Reduction, Source}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::ColumnRef; +use crate::ir::schema::series_identity::with_promql_series_identity; +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::ir::{ + ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, ScalarExpr, TimeRangeKind, +}; +use crate::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + +fn node(operator: Operator) -> Rc { + OperatorNode::new_shared(operator).unwrap() +} + +/// `topk by (job) (10, sum_over_time(m[1m]))` over series whose identity +/// is realized: the exact top-k and a heap's ranked readout of the same +/// rows (a `Project` naming the estimate's columns as the top-k's) +/// derive the same unique keys, one row per (job, series). +#[test] +fn exact_topk_and_heap_readout_derive_the_same_unique_keys() { + let open = Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, true), + ], + 0, + vec![], + ); + let rows = with_promql_series_identity(&node(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: open, + }))) + .unwrap(); + let window = node(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: rows, + })); + let per_series = node(Operator::NonASAP(NonASAPOp::Aggregate { + child: Rc::clone(&window), + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + })); + let exact = node(Operator::NonASAP(NonASAPOp::Aggregate { + child: per_series, + reduction: Reduction::by(vec![2]), + measures: vec![AggIntent::TopK { k: 10 }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + })); + let heap = node(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap), + input: SummaryUpdate { + item: Some(super::series_identity_item(&window.schema, &[2])), + weight: crate::summary_type::SummaryInputExpr::Column(ColumnRef::SampleValue), + }, + reduction: Reduction::by(vec![2]), + filter: None, + child: window, + })); + let estimate = node(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: heap, + query: SketchStatistic::TopK { k: 10 }, + })); + let estimated = estimate.schema.fields.len() - 1; + let cols = exact + .schema + .fields + .iter() + .enumerate() + .map(|(index, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(if index == 1 { + estimated + } else { + estimate + .schema + .fields + .iter() + .position(|input| input.name == field.name) + .unwrap() + }), + }) + .collect(); + let readout = node(Operator::NonASAP(NonASAPOp::Project { + cols, + child: estimate, + })); + assert!(!exact.schema.unique_keys.is_empty()); + assert_eq!(exact.schema.unique_keys, readout.schema.unique_keys); +} diff --git a/crates/types/src/stages/logical/group/mod.rs b/crates/types/src/stages/logical/group/mod.rs index 12b5a9bf9..8df0e7201 100644 --- a/crates/types/src/stages/logical/group/mod.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -258,15 +258,5 @@ pub fn ranked_column_of_group(memo: &Memo, group: MemoGroupId) -> Result Option { - crate::summary_type::raw_data_samples_of_metric(&MemoGroupAsInput { - memo, - group_id: group, - }) -} - #[cfg(test)] mod tests; diff --git a/crates/types/src/stages/logical/group/tests.rs b/crates/types/src/stages/logical/group/tests.rs index 59878ff41..e309825cb 100644 --- a/crates/types/src/stages/logical/group/tests.rs +++ b/crates/types/src/stages/logical/group/tests.rs @@ -1,12 +1,11 @@ //! The group-scope helpers over a memo group: the column a top-k ranks -//! ([`ranked_column_of_group`]) and the metric whose raw data samples a -//! group outputs ([`raw_data_samples_of_metric_of_group`]). +//! ([`ranked_column_of_group`]). use super::*; use crate::ir::operator::arguments::Reduction; use crate::ir::operator::AggIntent; use crate::ir::schema::DataType; -use crate::test_support::memo_fixtures::{aggregate, range, scan}; +use crate::test_support::memo_fixtures::{aggregate, scan}; use crate::workload::DataWorkload; /// A top-k over a one-measure grouped aggregate ranks its value, the last @@ -27,25 +26,3 @@ fn ranked_column_of_group_is_the_measure_or_the_sample_value() { assert_eq!(ranked_column_of_group(&memo, by_job), Ok(1)); assert_eq!(ranked_column_of_group(&memo, rows), Ok(1)); } - -/// The raw data samples of `m` reach a group through a range and a plain -/// sum, not through a rate. -#[test] -fn raw_data_samples_of_metric_of_group_follows_ranges_and_plain_sums() { - let mut memo = Memo::new(&DataWorkload::default()); - let rows = memo.find_or_add_group(scan("m", &[], vec![])).unwrap(); - let window = memo.find_or_add_group(range(60_000, rows)).unwrap(); - let summed = memo - .find_or_add_group(aggregate( - Reduction::PerEntity, - AggIntent::Sum { col: None }, - window, - )) - .unwrap(); - let rated = memo - .find_or_add_group(aggregate(Reduction::PerEntity, AggIntent::Rate, window)) - .unwrap(); - let metric = |group| raw_data_samples_of_metric_of_group(&memo, group); - assert_eq!(metric(summed), Some("m".to_string())); - assert_eq!(metric(rated), None); -} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index b546a477c..fe018bb8d 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -22,9 +22,8 @@ pub use candidate::{ }; pub use edge::LogicalEdge; pub use group::{ - ranked_column_of_group, raw_data_samples_of_metric_of_group, AggregateCoverage, - AggregateDefinition, ColumnLineage, CoverageError, GroupProperties, MemoGroup, MemoGroupError, - MemoGroupId, ValueSign, + ranked_column_of_group, AggregateCoverage, AggregateDefinition, ColumnLineage, CoverageError, + GroupProperties, MemoGroup, MemoGroupError, MemoGroupId, ValueSign, }; pub use memo::{Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index ba1accd63..1c70ea118 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -14,12 +14,12 @@ mod physical; mod pruning; pub use logical::{ - listed_build_parameters, ranked_column_of_group, raw_data_samples_of_metric_of_group, - AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, - CandidateForMemoGroup, CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, - LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, - LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, - OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, + listed_build_parameters, ranked_column_of_group, AggregateCoverage, AggregateDefinition, + CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, + ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, + LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, + MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, + ValueSign, }; pub use physical::{ output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, diff --git a/crates/types/src/stages/physical/edge.rs b/crates/types/src/stages/physical/edge.rs index 07f30e68b..d87e5d6da 100644 --- a/crates/types/src/stages/physical/edge.rs +++ b/crates/types/src/stages/physical/edge.rs @@ -18,3 +18,11 @@ pub struct PhysicalEdge { /// $\alpha^{\mathrm{phys}}_{u,c,i}$. pub satisfies_physical_requirement: bool, } + +impl PhysicalEdge { + /// Whether a plan may use the edge: $\alpha^{\log}_{u,c,i} = + /// \alpha^{\mathrm{phys}}_{u,c,i} = 1$. + pub fn is_logically_and_physically_admissible(&self) -> bool { + self.satisfies_logical_requirement && self.satisfies_physical_requirement + } +} diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs index 523021a54..79fabb86f 100644 --- a/crates/types/src/stages/physical/memo_candidates/mod.rs +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -43,13 +43,26 @@ impl PhysicalASAPMemoCandidates { edges: Vec, pruned_candidates: Vec>, ) -> Self { - let _ = ( + let mut admissible_edges_into_input: HashMap<_, Vec> = HashMap::new(); + let mut admissible_edges_out_of_candidate = vec![Vec::new(); physical_candidates.len()]; + for (id, edge) in edges.iter().enumerate() { + if edge.is_logically_and_physically_admissible() { + admissible_edges_into_input + .entry((edge.consumer_candidate_id, edge.input_index_of_consumer)) + .or_default() + .push(PhysicalEdgeId(id)); + admissible_edges_out_of_candidate[edge.producer_candidate_id.0] + .push(PhysicalEdgeId(id)); + } + } + Self { logical_candidates, physical_candidates, - edges, pruned_candidates, - ); - todo!("#1207 PR 5: index the admissible edges") + edges, + admissible_edges_into_input, + admissible_edges_out_of_candidate, + } } /// $\mathcal E$: every edge, in (consumer, input) order. @@ -68,18 +81,14 @@ impl PhysicalASAPMemoCandidates { consumer: PhysicalCandidateId, input_index_of_consumer: usize, ) -> &[PhysicalEdgeId] { - let _ = ( - consumer, - input_index_of_consumer, - &self.admissible_edges_into_input, - ); - todo!("#1207 PR 5") + self.admissible_edges_into_input + .get(&(consumer, input_index_of_consumer)) + .map_or(&[], Vec::as_slice) } /// The admissible edges out of `producer`: every consumer that may share it. pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { - let _ = (producer, &self.admissible_edges_out_of_candidate); - todo!("#1207 PR 5") + &self.admissible_edges_out_of_candidate[producer.0] } } diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 723104d35..8961595ff 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -122,20 +122,18 @@ fn candidates( memo: Memo, evaluation_grid_of_group: Vec>, ) -> PhysicalASAPMemoCandidates { - PhysicalASAPMemoCandidates { - logical_candidates: LogicalASAPMemoCandidates { + PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { memo, edges: vec![], pruned: vec![], evaluation_grid_of_group, exact_only_groups_of_query: vec![], }, - physical_candidates: vec![], - pruned_candidates: vec![], - edges: vec![], - admissible_edges_into_input: HashMap::new(), - admissible_edges_out_of_candidate: vec![], - } + vec![], + vec![], + vec![], + ) } fn candidate( @@ -302,3 +300,90 @@ fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime } } } + +/// The edge from candidate `producer` into input 0 of `consumer`. +fn physical_edge( + producer: usize, + consumer: usize, + satisfies_logical_requirement: bool, + satisfies_physical_requirement: bool, +) -> PhysicalEdge { + PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(producer), + consumer_candidate_id: PhysicalCandidateId(consumer), + input_index_of_consumer: 0, + satisfies_logical_requirement, + satisfies_physical_requirement, + } +} + +/// An edge is admissible only when it satisfies both the logical and the +/// physical requirement. +#[test] +fn an_edge_is_admissible_when_both_requirements_hold() { + for (logical, physical) in [(false, false), (false, true), (true, false), (true, true)] { + assert_eq!( + physical_edge(0, 1, logical, physical).is_logically_and_physically_admissible(), + logical && physical, + "alpha_log {logical}, alpha_phys {physical}" + ); + } +} + +/// `new` indexes only the admissible edges, into each (consumer, input) +/// and out of each producer; every edge stays in `edges()`. The last +/// 5 minutes read by a query-time KLL build (admissible), by an +/// ingestion-time chain (not physically admissible) and by a query-time +/// chain that misses the logical requirement. +#[test] +fn new_indexes_only_admissible_edges() { + let physical_candidates = vec![ + candidate(LAST_5M, QueryTime, Lifetime::None), + candidate(BUILD, QueryTime, Lifetime::None), + candidate( + WINDOWS_5S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ), + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + ]; + let edges = vec![ + physical_edge(0, 1, true, true), + physical_edge(0, 2, true, false), + physical_edge(0, 3, false, true), + ]; + let candidates = PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo: memo(), + edges: vec![], + pruned: vec![], + evaluation_grid_of_group: vec![None; 5], + exact_only_groups_of_query: vec![], + }, + physical_candidates, + edges.clone(), + vec![], + ); + assert_eq!( + candidates.edges().map(|(_, edge)| edge).collect::>(), + edges + ); + assert_eq!( + candidates.admissible_edges_into(PhysicalCandidateId(1), 0), + [PhysicalEdgeId(0)] + ); + for consumer in [2, 3] { + assert!(candidates + .admissible_edges_into(PhysicalCandidateId(consumer), 0) + .is_empty()); + } + assert_eq!( + candidates.admissible_edges_out_of(PhysicalCandidateId(0)), + [PhysicalEdgeId(0)] + ); + for producer in 1..4 { + assert!(candidates + .admissible_edges_out_of(PhysicalCandidateId(producer)) + .is_empty()); + } +} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index 6a8fdeef4..d5b4227fb 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -113,12 +113,14 @@ pub enum PruneReason { /// Stage 2: it runs at ingestion time, and the source has no ingestion /// pipeline (its data is at rest or its arrival unknown). NoIngestionPipeline, - /// Stage 2: it scans raw data at query time, and the deployment does - /// not retain raw data, so no window of it is there to scan. + /// Stage 2: it scans raw data at query time, or retains the raw rows it + /// scans across evaluations, and the deployment does not retain raw + /// data, so no window of it is there to scan or keep. RawDataNotRetained, - /// Stage 2: it is a query-time result materialized (`OneEvaluation` or - /// `AcrossEvaluations`), and the deployment retains no query-time - /// results. + /// Stage 2: it is a query-time result materialized across evaluations + /// (`AcrossEvaluations`), and the deployment retains no query-time + /// result across evaluations + /// (`DeploymentCapabilities::query_time_results_retained`). QueryTimeResultsNotRetained, /// Stage 2: one of its inputs has no producer candidate that can be /// built: none is left, or every one depends, directly or through a diff --git a/crates/types/src/summary_type/mod.rs b/crates/types/src/summary_type/mod.rs index 1ce1ec778..9cc98adb5 100644 --- a/crates/types/src/summary_type/mod.rs +++ b/crates/types/src/summary_type/mod.rs @@ -6,6 +6,7 @@ mod capability; mod parameter_lists; mod parameters; mod statistic; +mod update; /// A specific sketch algorithm — bounded error, sized by its /// [`SketchParams`]. It is `asap_sketchlib`'s family enum; the statistics @@ -21,7 +22,5 @@ pub use parameters::{ ExactKind, ExactParams, HydraParams, SamplingKind, SamplingParams, SketchParams, StatModelKind, StatModelParams, WaveletKind, WaveletParams, EXACT_ACCUMULATOR_STATE_BYTES, KLL_MAX_K, }; -pub use statistic::{ - raw_data_samples_of_metric, EntityIdentity, NonNegativeWeightProof, SketchStatistic, - SummaryInputExpr, SummaryUpdate, WeightDomain, -}; +pub use statistic::SketchStatistic; +pub use update::{EntityIdentity, SummaryInputExpr, SummaryUpdate}; diff --git a/crates/types/src/summary_type/statistic/mod.rs b/crates/types/src/summary_type/statistic/mod.rs index 47a7e3896..0912ceb2d 100644 --- a/crates/types/src/summary_type/statistic/mod.rs +++ b/crates/types/src/summary_type/statistic/mod.rs @@ -1,114 +1,9 @@ -//! What a summary is updated with and the statistics read from it. +//! The statistics read from a summary state. use serde::{Deserialize, Serialize}; -use crate::ir::operator::arguments::Source; -use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; -use crate::ir::{NonASAPOp, Operator}; - -/// One summary-state update. `item` identifies a keyed item when the family -/// is keyed; `weight` is the observation/update expression applied to state. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SummaryUpdate { - pub item: Option, - pub weight: SummaryInputExpr, - /// Planner proof for algorithms whose error/update semantics require a - /// non-negative stream. Absence of proof is never interpreted as zero or - /// non-negative. - #[serde(default)] - pub weight_domain: WeightDomain, -} - -impl SummaryUpdate { - pub fn column(column: ColumnRef) -> Self { - Self { - item: None, - weight: SummaryInputExpr::Column(column), - weight_domain: WeightDomain::UnknownOrSigned, - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum WeightDomain { - #[default] - UnknownOrSigned, - NonNegative { - proof: NonNegativeWeightProof, - }, - /// An open requirement, not a proof: the weights are raw data samples of - /// `metric`, or plain sums of them (no rate, no other arithmetic), so they - /// are non-negative when every raw data sample of `metric` is: `metric` is - /// declared a counter, or declared non-negative. Logical optimization - /// records it without reading the data workload; plan selection confirms - /// it from the declared source facts - /// ([`SourceFacts::metric_type`](crate::workload::SourceFacts::metric_type), - /// [`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). - /// Unconfirmed, it counts as not proven. Serialized as - /// `samples_of_metric`. - #[serde(rename = "samples_of_metric")] - RawDataSamplesOfMetric { - metric: String, - }, - /// An open requirement, not a proof: the weights are reset-corrected - /// `rate`, `increase` or `irate` values of `metric`'s raw data samples, or plain - /// sums of them. Reset correction keeps them non-negative exactly when - /// the samples are, that is when `metric` is a counter; over a gauge a - /// negative sample can give a negative value. Plan selection confirms - /// it from the declared metric types, as - /// [`RawDataSamplesOfMetric`](Self::RawDataSamplesOfMetric). Unconfirmed, it - /// counts as not proven. - CounterDerivativeOfMetric { - metric: String, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum NonNegativeWeightProof { - UnitCount, - /// `rate`, `increase` or `irate` of a counter's samples, or plain sums - /// of them: reset correction over non-negative samples gives a - /// non-negative increase, and a rate divides it by a positive duration. - /// Plan selection gives it on confirming a - /// [`WeightDomain::CounterDerivativeOfMetric`] requirement. - ResetAwareCounterDerivative, - /// Raw data samples of a metric declared a counter - /// ([`PrometheusMetricType::Counter`](crate::workload::PrometheusMetricType::Counter)), or - /// plain sums of them: a counter sample is never negative. Plan selection - /// gives it on confirming a - /// [`WeightDomain::RawDataSamplesOfMetric`] requirement. Serialized as - /// `counter_samples`. - #[serde(rename = "counter_samples")] - CounterRawDataSamples, - /// Raw data samples of a metric declared non-negative - /// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), - /// such as a gauge, or plain sums of them. Plan selection gives it on - /// confirming a [`WeightDomain::RawDataSamplesOfMetric`] requirement for a - /// metric not declared a counter. Serialized as - /// `declared_non_negative_samples`. - #[serde(rename = "declared_non_negative_samples")] - DeclaredNonNegativeRawDataSamples, - /// A constant weight greater than 0, such as `Constant(2.0)`: every row - /// adds the same positive amount, so no weight is negative. - PositiveConstant, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum EntityIdentity { - PromqlLabelSet { excluding: Vec }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum SummaryInputExpr { - Constant(f64), - Column(ColumnRef), - Tuple(Vec), - EntityIdentity(EntityIdentity), -} /// The statistic computed from summary state by `SummaryEstimate`, for example /// `Quantile { q: 0.99 }`. This is a result operation, not a workload query. @@ -157,34 +52,5 @@ impl SketchStatistic { } } -/// The metric whose raw data samples, or plain sums of them, are every value -/// `input` outputs, or `None`: walking down through a time range, an -/// `offset`-only time shift (it moves the window, not the samples) and a -/// plain `sum`, to a time-series scan. Any other operator (rate and -/// arithmetic included) ends it. The metric's type is not read here: whether -/// its samples are non-negative is plan selection's question -/// ([`WeightDomain::RawDataSamplesOfMetric`]). -pub fn raw_data_samples_of_metric(input: &I) -> Option { - let Operator::NonASAP(operator) = input.operator() else { - return None; - }; - match operator { - NonASAPOp::Scan { - source: Source::TimeSeries { metric }, - .. - } => Some(metric), - NonASAPOp::TimeRange { child, .. } => raw_data_samples_of_metric(&child), - NonASAPOp::TimeShift { shift, child } if shift.at.is_none() => { - raw_data_samples_of_metric(&child) - } - NonASAPOp::Aggregate { - measures, child, .. - } if matches!(measures.as_slice(), [AggIntent::Sum { col: None }]) => { - raw_data_samples_of_metric(&child) - } - _ => None, - } -} - #[cfg(test)] mod tests; diff --git a/crates/types/src/summary_type/statistic/tests.rs b/crates/types/src/summary_type/statistic/tests.rs index b485d6050..484da2cf2 100644 --- a/crates/types/src/summary_type/statistic/tests.rs +++ b/crates/types/src/summary_type/statistic/tests.rs @@ -1,124 +1,22 @@ use super::*; -use crate::ir::operator::arguments::Reduction; -use crate::ir::schema::{DataType, Field, Schema}; -use crate::ir::{OperatorNode, TimeRangeKind}; -use std::rc::Rc; -use std::time::Duration; -/// The raw data samples of `m` reach a node through a range, an -/// `offset`-only time shift and a plain `sum`; a rate ends them. #[test] -fn raw_data_samples_of_metric_follows_ranges_shifts_and_plain_sums() { - let node = |operator: Operator>| OperatorNode::new_shared(operator).unwrap(); - let rows = node(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })); - let window = node(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: rows, - })); - let per_series = |measure: AggIntent| { - node(Operator::NonASAP(NonASAPOp::Aggregate { - child: Rc::clone(&window), - reduction: Reduction::PerEntity, - measures: vec![measure], - output_names: vec![String::new()], - filters: vec![], - having: None, - })) - }; - assert_eq!(raw_data_samples_of_metric(&window), Some("m".to_string())); - let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { - shift: crate::ir::operator::arguments::TimeShift { - offset_ms: 60_000, - at: None, - }, - child: Rc::clone(&window), - })); - assert_eq!(raw_data_samples_of_metric(&shifted), Some("m".to_string())); +fn topk_statistic_round_trips() { + let query = SketchStatistic::TopK { k: 10 }; + let json = serde_json::to_string(&query).unwrap(); assert_eq!( - raw_data_samples_of_metric(&per_series(AggIntent::Sum { col: None })), - Some("m".to_string()) - ); - assert_eq!( - raw_data_samples_of_metric(&per_series(AggIntent::Rate)), - None + serde_json::from_str::(&json).unwrap(), + query ); } -#[test] -fn keyed_summary_input_and_topk_evaluation_round_trip() { - for input in [ - SummaryUpdate { - item: Some(SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { excluding: vec![] }, - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("service".into()), - )])), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }, - ] { - let input_json = serde_json::to_string(&input).unwrap(); - assert_eq!( - serde_json::from_str::(&input_json).unwrap(), - input - ); - let query = SketchStatistic::TopK { k: 10 }; - let json = serde_json::to_string(&query).unwrap(); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - query - ); - } -} - -/// The raw-data-sample and summary-sample renames keep their JSON names, -/// so committed plans and workloads still read. +/// The summary-sample rename keeps its JSON name, so committed plans +/// and workloads still read. #[test] fn renamed_sample_terms_keep_their_wire_names() { use crate::ir::schema::FieldDataType; use crate::summary_type::{SamplingKind, SamplingParams}; let json = |value: serde_json::Value| value.to_string(); - let raw = WeightDomain::RawDataSamplesOfMetric { - metric: "m".to_string(), - }; - assert_eq!( - serde_json::to_string(&raw).unwrap(), - json(serde_json::json!({"kind": "samples_of_metric", "metric": "m"})) - ); - for (proof, name) in [ - ( - NonNegativeWeightProof::CounterRawDataSamples, - "counter_samples", - ), - ( - NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, - "declared_non_negative_samples", - ), - ] { - assert_eq!( - serde_json::to_string(&proof).unwrap(), - format!("\"{name}\"") - ); - } let summary_sample = FieldDataType::SummarySample( SamplingKind::Reservoir, SamplingParams::Reservoir { size: 64 }, diff --git a/crates/types/src/summary_type/update/mod.rs b/crates/types/src/summary_type/update/mod.rs new file mode 100644 index 000000000..39593666d --- /dev/null +++ b/crates/types/src/summary_type/update/mod.rs @@ -0,0 +1,38 @@ +//! What a summary state is updated with, per row. + +use serde::{Deserialize, Serialize}; + +use crate::ir::scalar::ColumnRef; + +/// One summary-state update. `item` identifies a keyed item when the family +/// is keyed; `weight` is the observation/update expression applied to state. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SummaryUpdate { + pub item: Option, + pub weight: SummaryInputExpr, +} + +impl SummaryUpdate { + pub fn column(column: ColumnRef) -> Self { + Self { + item: None, + weight: SummaryInputExpr::Column(column), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EntityIdentity { + PromqlLabelSet { excluding: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum SummaryInputExpr { + Constant(f64), + Column(ColumnRef), + Tuple(Vec), + EntityIdentity(EntityIdentity), +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/summary_type/update/tests.rs b/crates/types/src/summary_type/update/tests.rs new file mode 100644 index 000000000..d583514d9 --- /dev/null +++ b/crates/types/src/summary_type/update/tests.rs @@ -0,0 +1,25 @@ +use super::*; + +#[test] +fn keyed_summary_update_round_trips() { + for input in [ + SummaryUpdate { + item: Some(SummaryInputExpr::EntityIdentity( + EntityIdentity::PromqlLabelSet { excluding: vec![] }, + )), + weight: SummaryInputExpr::Constant(1.0), + }, + SummaryUpdate { + item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( + ColumnRef::Named("service".into()), + )])), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + }, + ] { + let input_json = serde_json::to_string(&input).unwrap(); + assert_eq!( + serde_json::from_str::(&input_json).unwrap(), + input + ); + } +} diff --git a/docs/design_docs/architecture/README.md b/docs/design_docs/architecture/README.md index a521d7e3f..63f713f83 100644 --- a/docs/design_docs/architecture/README.md +++ b/docs/design_docs/architecture/README.md @@ -47,7 +47,7 @@ target. | Query frontends | `frontend-sql`, `frontend-promql`, `frontend-metricsql` | Stage 0: parse source languages and produce canonical Pre-ASAP queries | | Stage 0 hash-consing | `stage0-memo` | The memo groups of the workload's query roots | | Logical ASAP-aware optimization | `stage1-pass1-per-computation-candidates`, `stage1-pass2-reuse-candidates` | Stage 1: per-computation candidates (Pass 1), ASAP-aware CSE (Pass 2) | -| Physical ASAP-aware optimization | `stage2-physical-optimization` | Stage 2: materialization and physical operator implementation | +| Physical ASAP-aware optimization | `stage2-physical-optimization` | Stage 2: each logical candidate's timing and lifetime, the physical edges and the pruned alternatives | | Plan selection | `stage3-plan-selection` | Stage 3: accuracy and latency checks, costing and selection; the planning models and the Stage 1 to Stage 3 driver | | Planner facade | `planner` | `e2e_plan`: Stage 0 followed by Stages 1–3 | | Executor | `executor` | Stage 4: runs a `PhysicalASAPDAG` over `asap_sketchlib` summaries; no stage crate depends on it | diff --git a/docs/design_docs/concepts/glossary.md b/docs/design_docs/concepts/glossary.md index 8b52e6ef6..0e8ccfc15 100644 --- a/docs/design_docs/concepts/glossary.md +++ b/docs/design_docs/concepts/glossary.md @@ -42,7 +42,7 @@ A node of the physical DAG whose output is materialized across evaluations (life ## Raw data sample -One timestamped value of a time series as it arrives, such as one Prometheus Remote Write sample. Raw data samples are the rows a time-series scan reads. Code names them with `raw_data_sample`, e.g. `WeightDomain::RawDataSamplesOfMetric`, `raw_data_samples_of_metric`. +One timestamped value of a time series as it arrives, such as one Prometheus Remote Write sample. Raw data samples are the rows a time-series scan reads. Code names them with `raw_data_sample`, e.g. `RawDataWindow`. ## Summary sample diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index eac375728..174d89c79 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -1254,13 +1254,13 @@ This part of the code answers two questions about summary state: **Update input examples** (as the planner builds them in Stage 1, `asap-stage1-pass1-per-computation-candidates`): -| Query | Sketch | `item` | `weight` | `weight_domain` | -|---|---|---|---|---| -| p99 of `latency` | KLL | none: KLL has no keys | `Column(latency)`: the value itself | `UnknownOrSigned` | -| how often each `endpoint` occurs | Count-Min | `Column(endpoint)` | `Constant(1.0)`: each row counts once | `NonNegative(UnitCount)` | -| `topk(5, rate(http_requests_total[5m]))` | Count-Min with heap | `Tuple(every column except value and the group columns)`: one item per series, including its timestamp | `Column(value)`: the rate | `NonNegative(ResetAwareCounterDerivative)` | +| Query | Sketch | `item` | `weight` | +|---|---|---|---| +| p99 of `latency` | KLL | none: KLL has no keys | `Column(latency)`: the value itself | +| how often each `endpoint` occurs | Count-Min | `Column(endpoint)` | `Constant(1.0)`: each row counts once | +| `topk(5, rate(http_requests_total[5m]))` | Count-Min with heap | `Tuple(every column except value and the group columns)`: one item per series, including its timestamp | `Column(value)`: the rate | -`weight_domain` matters because some sketches (e.g. Count-Min) are only accurate when weights are never negative. The planner records why a weight is non-negative; if it cannot prove it, the weight counts as possibly negative. +Some sketches (e.g. Count-Min) are only accurate when weights are never negative. The sign of the weights is not part of the update: it is the state group's `column_signs` entry for its state column, derived from the input groups and the declared source signs, and Stage 2 prunes a Count-Min build whose weights are not proven non-negative (`CountMinWeightsNotProvenNonNegative`). **Evaluation examples:** `Quantile { q: 0.99 }` reads p99 from a KLL; `PointCount { key: endpoint, value: Some("checkout") }` reads how often `checkout` occurred from a Count-Min; `TopK { k: 5 }` reads the 5 heaviest items. @@ -1273,25 +1273,11 @@ pub struct SummaryUpdate { /// The value added to the state: the observed value for a KLL, or the /// count to add for a Count-Min (often `Constant(1.0)`). pub weight: SummaryInputExpr, - /// Whether `weight` is proven never negative. Some algorithms need that. - /// Missing proof is never treated as non-negative. - pub weight_domain: WeightDomain, } impl SummaryUpdate { - /// Add the value of column `c` for each row: no item, and the weight is - /// not known to be non-negative. + /// Add the value of column `c` for each row, with no item. pub fn column(c: ColumnRef) -> Self; } -pub enum WeightDomain { - /// Nothing is known; the weight may be negative. The default. - UnknownOrSigned, - /// The weight is never negative. `proof`: why. - NonNegative { proof: NonNegativeWeightProof }, -} -pub enum NonNegativeWeightProof { - UnitCount, // every row adds 1 - ResetAwareCounterDerivative, // PromQL increase/rate over counters with reset correction -} /// An expression that computes an item or a weight from the input row. pub enum SummaryInputExpr { Constant(f64), // the same number for every row diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 91a950b41..93faa0821 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -543,16 +543,14 @@ The sections below describe each stage. | 0. Frontends | nothing: a query that cannot be lowered fails planning with an error | — | — | | 1, Pass 1 | nothing is recorded: a way the measure or the deployment cannot compute is not offered (the exact accumulator of an `avg` over a nullable input, a count sketch over relational rows, a summary the deployment does not support, a quantile summary outside [0, 1], a top-k heap over one row per item), nor a way Pass 1 cannot build | — | — | | 1, Pass 2 | nothing: it only adds candidates | — | — | -| 2 | what cannot be built or materialized: `NoIngestionPipeline`, `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `QueryTimeResultsNotRetained`, `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`; the planner does not check executor support | `PhysicalASAPMemoCandidates::pruned_candidates` | the `PhysicalCandidate` it would have been | +| 2 | what cannot be built or materialized: `NoIngestionPipeline`, `RawDataNotRetained` (a scan of raw data the deployment does not keep, at query time or materialized across evaluations), `QueryTimeResultsNotRetained`, `CountMinWeightsNotProvenNonNegative`, `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`; the planner does not check executor support | `PhysicalASAPMemoCandidates::pruned_candidates` | the `PhysicalCandidate` it would have been | | 3 | valid candidates that lose, and the checks only a whole plan can make (accuracy, latency, cluster capacity) | its own `RejectionCause` | — | -Stage 2's key names what a pruned candidate would have been, since kept -candidates are renumbered after pruning and a pruned one has no -`PhysicalCandidateId`. Stage 2 emits one candidate per operator node, -timing and lifetime, and makes no copies of it, so (`operator_node`, -`properties`) is unique. Operator -nodes are kept whole, so `operator_node` indexes the output's -`operator_nodes_of_candidates_for_memo_groups`. Pass 1 records no pruned +Stage 2 records a pruned candidate as the `PhysicalCandidate` it would +have been, since kept candidates are renumbered after pruning and a pruned +one has no `PhysicalCandidateId`. Stage 2 emits one candidate per logical +candidate, timing and lifetime, and makes no copies of it, so +(`logical_candidate_id`, `physical_properties`) is unique. Pass 1 records no pruned candidate: a way it cannot compute is not offered. `asap_planner` writes both lists as `pruned_candidates` under @@ -573,7 +571,7 @@ the memo path's entry points. | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (no longer in the code) | Logical roots → Stage 0's `Memo` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support → the same `Memo` with each group's `CandidateForMemoGroup`s (a summary with one build per configuration of its `parameter_list`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → `LogicalASAPMemoCandidates`: the same memo with the rules' groups and candidates added (rollup, top-k reuse, chain groups), each readout's required accuracy, the logical edges between the surviving candidates, the pruned candidates and, per query, the groups it may read only exactly | `generate_logical_asap_memo_candidates` (`asap_stage1_pass2_reuse_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates` (with each group's evaluation grid); `DataWorkload` (arrival), `DeploymentCapabilities` (raw-data and query-time retention) → `PhysicalASAPMemoCandidates`: every logical candidate Pass 2 kept with each offered timing and lifetime (one physical candidate each, no copies), every logical edge expanded into physical edges with $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_stage3_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -1261,14 +1259,17 @@ Output: C3 — CandidatePhysicalASAPDAGs here; they determine the cost of each option, which stage 3 uses to pick the cheapest plan (the typical outcomes above). -On the memo, Stage 2 prunes per operator node and records each pruned -candidate in `PhysicalASAPMemoCandidates::pruned_candidates` +On the memo, Stage 2 prunes per physical alternative (a logical candidate +with one timing and lifetime) and records each pruned alternative in `PhysicalASAPMemoCandidates::pruned_candidates` ([Pruning by stage](#pruning-by-stage)). It does not generate a configuration the deployment cannot run: an ingestion-time alternative -without an ingestion pipeline (`NoIngestionPipeline`), a query-time scan -of raw data the deployment does not retain (`RawDataNotRetained`), and a -materialized query-time result the deployment does not retain -(`QueryTimeResultsNotRetained`). It does not check executor support. It +without an ingestion pipeline (`NoIngestionPipeline`), a scan of raw data +the deployment does not retain, at query time or materialized across +evaluations (`RawDataNotRetained`), a +query-time result materialized across evaluations the deployment does not +retain (`QueryTimeResultsNotRetained`), and a Count-Min build whose weights +are not proven non-negative (`CountMinWeightsNotProvenNonNegative`). It does +not check executor support. It then removes the candidates no buildable input or query output can use (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`), repeating until nothing more is removed. Stage 2 makes no execution-scope diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index f3bf869c6..5d974ced4 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -23,7 +23,7 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | `asap-stage1-pass1-per-computation-candidates` (`crates/stage1-pass1-per-computation-candidates`) | Stage 1, Pass 1: the initial candidates of each memo group | | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates`) | Stage 1, Pass 2: the reuse rules over memo groups and the logical edges | | `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, each operator's error contract, the target comparison `within_limit`), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`ErrorMetricsOfMemo::edge_meets_logical_requirements`); depends only on `asap-types` | -| `asap-stage2-physical-optimization` (`crates/stage2-physical-optimization`) | Stage 2: materialization and physical operator implementation | +| `asap-stage2-physical-optimization` (`crates/stage2-physical-optimization`) | Stage 2: each logical candidate's timing and lifetime, the physical edges and the pruned alternatives; depends only on `asap-types` | | `asap-cost` (`crates/cost`) | The cost model the stages share: resource-usage estimation (the analytical per-operator model, measured operation costs, state byte sizes) and the cost models (`ResourceUsage`, `CostModel`, `CostInputs`); depends only on `asap-types` | | `asap-stage3-plan-selection` (`crates/stage3-plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | | `asap-planner` (`crates/planner`) | The facade: `e2e_plan` runs Stage 0, then the optimization pass (`Stage1ToStage3Pass` by default), which orchestrates Stages 1–3 (`run_planning`); the planning models (`PlanningModels`) and the `OptimizationPass` trait | @@ -76,13 +76,13 @@ same pipeline behind the `OptimizationPass` trait. | Pass 2 steps: rollup from the finest common group (shared producers across group keys), top-k reuse, readout requirements, dead-candidate elimination, chain groups (tumbling windows) and the logical edges | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates/src/`): `lib.rs` (the steps in order), `rules/` (`candidates.rs`, `rollup/`, `topk.rs`, `chains.rs`), `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs` and `evaluation_grids.rs`; the groups each query may read only exactly come from `asap_accuracy::ErrorMetricsOfMemo` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | none today (the standalone enumeration was deleted) | | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | -| Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-stage2-physical-optimization` | +| Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes in `asap-types`, `stages/physical/memo_candidates/mod.rs` | | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | | Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-stage3-plan-selection`, `lib.rs`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`) and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | | The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | | The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | -| What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | +| What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained`; Stage 3 reads none of it | | The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `models.rs` (`PlanningModels`) and `pass.rs` (the trait); the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | The current production search organizes these computations in memo result/state @@ -104,10 +104,10 @@ The unwired maintained-population rule has been removed as well; no Stage 1 pass For each composed DAG, the Pass 2 rules find what can be shared, split into tumbling windows or reused, and each subset of their candidates becomes another Stage 1 candidate. -3. **Stage 2** gives each summary build its physical properties options and emits - one physical candidate per assignment (the all-not-materialized one - first), skipping assignments that put a build at ingestion time over one - that is not. +3. **Stage 2** expands each logical candidate into one physical candidate + per offered timing and lifetime, each logical edge into the physical + edges between their alternatives, and records the alternatives it + prunes. 4. **Stage 3** checks each physical candidate: every query's root guarantee against its target (binding the statistics Stage 1 left symbolic from the `DataWorkload`), every query's latency against `max_latency_ms` and, for a diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 2de5b59f2..6ceb70f65 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -77,30 +77,27 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- ## 4. Stage 2 -- Only summary builds (`SummaryAgg`, `SummaryTumblingWindowChain`) get materialization - options, each a timing and a lifetime (`PhysicalProperties`): not materialized always; - materialized at ingestion time, `AcrossEvaluations(Memory)`, when the build is - eligible to run as data arrives; materialized at query time for `OneEvaluation(Memory)` - when two or more queries read it (reused within one evaluation); materialized at - query time `AcrossEvaluations(Memory)` for a tumbling-window chain that - builds one new tumbling window per evaluation, when the deployment declares - `query_time_results_retained`. -- One `PhysicalCandidate` per assignment, the all-not-materialized one first - (`P`, then `P-m`). An assignment that puts a build at ingestion - time over one that is not is not generated. A logical candidate Stage 2 - cannot implement becomes a `PrunedCandidate` with its reason. -- On the memo (`generate_physical_asap_memo_candidates`), every physical - candidate Stage 2 prunes is recorded in +- Each logical candidate Pass 2 kept gets one `PhysicalCandidate` per + offered timing and lifetime (`PhysicalProperties`): any candidate, not + only a summary build, may get a lifetime. `IngestionTime` is offered to + a time-independent group and its inputs (a counter state only over + ordered rows); `AcrossEvaluations` to a chain, a group on a known + evaluation grid and a time-independent group, unless the group is read + under an absolute `@ T`; (IngestionTime, None) not to a summary build. A + value not offered is not recorded. +- Every physical alternative Stage 2 prunes is recorded in `PhysicalASAPMemoCandidates::pruned_candidates` as `PrunedCandidate` with a `PruneReason`: an ingestion-time alternative without an ingestion pipeline - (`NoIngestionPipeline`), a query-time scan of raw data the deployment does - not retain (`RawDataNotRetained`) and a materialized query-time result the - deployment does not retain (`QueryTimeResultsNotRetained`) are not - generated; a candidate no buildable input or query output can use is - removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`). The - planner does not check executor support. Stage 2 emits one candidate per operator node, - timing and lifetime; it makes no execution-scope copies. + (`NoIngestionPipeline`); a scan of raw data the deployment does not + retain, at query time or materialized across evaluations + (`RawDataNotRetained`); a query-time result materialized across + evaluations the deployment does not retain + (`QueryTimeResultsNotRetained`); a Count-Min build whose weights are not + proven non-negative (`CountMinWeightsNotProvenNonNegative`); then an + alternative no buildable input or query output can use + (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`). The planner does + not check executor support. Stage 2 makes no execution-scope copies. ## 5. Stage 3 diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 87cab901f..6b0d24b4d 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -81,10 +81,10 @@ contains the candidate and that Stage 3 prices it. (`crates/types/src/stages/physical/properties/mod.rs`, `PhysicalProperties::new` and `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. -2. Offer it in `crates/stage2-physical-optimization/src/lib.rs` (the - `MaterializedOutputCandidate` options), after the existing options so default - candidate ids stay the same, and gate it on the deployment input that - allows it. +2. Offer it in `crates/stage2-physical-optimization/src/candidate.rs` + (`LIFETIMES`, `offered_physical_properties`; a per-group offer in + `group.rs`), and gate it on the + deployment input that allows it with a `PruneReason`. 3. Price it in `crates/cost/src/node/usage.rs` (`asap_cost::node_usage`): what it keeps in memory or on disk, and what each evaluation rebuilds.