From 0f5a5f96026692acbc8c2d18f2488428bb53cab6 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:52:03 +0000 Subject: [PATCH 1/4] refactor: no silent defaults for kernel memory, operator rules and aggregate names An unknown stays unanswered until code that knows the safe direction handles it, and a match that returns a constant lists every operator, so a new operator fails to build instead of getting a default. - AggregateCore::approx_memory_bytes has no 4096-byte default; every kernel says what it holds (doc tests show a kernel without it fails to build). - A per-family test checks plan selection's state bytes against the executor's charge on seeded rows, within 1.5x, with a known-gap list (DDSketch, exact accumulators, HydraCms) naming each issue. - InputTimeAccess::of and windows_computed_from list every operator; behaviour unchanged, pinned by a per-operator table test. - Aggregate output_names is required when deserialized. - clippy::wildcard_enum_match_arm warns in types::plan::time_window and asap-cost node::usage; every wildcard there lists its variants. --- crates/cost/src/node/state_bytes/tests.rs | 297 ++++++++++++++- crates/cost/src/node/usage/mod.rs | 182 +++++++++- crates/executor/src/summary_kernels/traits.rs | 60 +++- crates/types/src/ir/operator/non_asap.rs | 31 +- crates/types/src/plan/time_window.rs | 339 +++++++++++++++++- 5 files changed, 873 insertions(+), 36 deletions(-) diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index 3e30a6bb2..17274e2fb 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -24,6 +24,32 @@ fn exact_state_is_priced_at_the_executors_state_size() { assert_eq!(asap_types::summary_type::EXACT_ACCUMULATOR_STATE_BYTES, 56); } +/// The rows a family is fed. +enum Rows { + /// One sample value a row. + Values, + /// One label value a row, as Pass 1 feeds UnivMon a column. + Labels, + /// A label set and a weight a row. + Keyed, +} + +/// A 16-byte label value, `i` zero-padded: its key has +/// [`ASSUMED_VALUE_KEY_BYTES`]. +fn label(i: u32) -> String { + format!("{i:016}") +} + +/// Three 16-byte labels, the first [`label`]`(i)`: a tuple key of +/// [`ASSUMED_HEAP_KEY_BYTES`]. +fn labels(i: u32) -> [String; 3] { + [ + label(i), + "service-frontend".to_string(), + "region-us-east-1".to_string(), + ] +} + /// The bytes `asap_sketchlib`'s DDSketch holds after adding `values`. fn library_ddsketch_bytes(alpha: f64, values: &[f64]) -> u64 { let mut sketch = asap_sketchlib::sketches::DdSketch::new(alpha); @@ -174,15 +200,6 @@ fn state_bytes_bound_the_executors_charge() { /// At least, and at most 1.5 times once the state is full. Bound, } - /// The rows a family is fed. - enum Rows { - /// Signed values over the assumed range. - Values, - /// One label value a row, as Pass 1 feeds UnivMon a column. - Labels, - /// A label set and a weight a row. - Keyed, - } // (family, parameters, rows fed, how it is priced, the // accumulator's own fields, rows at which the state is full: four // times the heap for the heap sketches, so each heap churns). @@ -301,20 +318,10 @@ fn state_bytes_bound_the_executors_charge() { item, weight: SummaryInputExpr::Constant(1.0), }; - // A 16-byte label value: its key has the assumed size. - let label = |i: u32| format!("{i:016}"); assert_eq!( Value::Utf8(label(0).into()).key().unwrap().len(), ASSUMED_VALUE_KEY_BYTES ); - // Three 16-byte labels: a tuple key of the assumed size. - let labels = |i: u32| { - [ - label(i), - "service-frontend".to_string(), - "region-us-east-1".to_string(), - ] - }; let key = asap_sketchlib::TupleKey::new( labels(0) .into_iter() @@ -394,3 +401,255 @@ fn state_bytes_bound_the_executors_charge() { } } } + +/// How far plan selection's bytes for a state may be from what the +/// executor charges for it in [`priced_bytes_agree_with_the_executor`]: +/// priced ÷ charged in `[1 / 1.5, 1.5]`. +const BYTES_AGREEMENT_FACTOR: f64 = 1.5; + +/// The families whose priced bytes are known to disagree with the +/// executor's beyond [`BYTES_AGREEMENT_FACTOR`]: (family, the issue that +/// fixes it, priced ÷ charged as measured by +/// [`priced_bytes_agree_with_the_executor`]). A family leaves the list +/// when its issue is fixed: the test fails while it is listed but agrees, +/// or when its ratio moves off the one listed. +const KNOWN_BYTES_GAPS: &[(&str, u32, f64)] = &[ + // Priced for values over nine decades (`DDSKETCH_ASSUMED_VALUE_RANGE`); + // these span three. + ("DDSketch", 1313, 2.982), + // Priced as one element of the executor's state column (56 bytes); + // a boxed accumulator charges its whole struct (136 bytes). + ("exact Sum accumulator", 1301, 0.412), + ("exact Count accumulator", 1301, 0.412), + ("exact Min accumulator", 1301, 0.412), + ("exact Max accumulator", 1301, 0.412), + ("exact Increase accumulator", 1301, 0.412), + ("exact Rate accumulator", 1301, 0.412), + // Priced as one Count-Min per group; the executor keeps one grid for + // all of them, here 3 × 64 cells over 10 groups. + ("HydraCms", 947, 0.104), +]; + +/// The next value of the seeded stream `state` (splitmix64), uniform in +/// `[0, 1)`, so the rows every family is fed are fixed. +fn next_unit(state: &mut u64) -> f64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + ((z ^ (z >> 31)) >> 11) as f64 / (1u64 << 53) as f64 +} + +/// Plan selection prices each state at what the executor charges for it +/// on data shaped like a workload's, not only on the worst case +/// [`state_bytes_bound_the_executors_charge`] feeds. Every family the +/// executor builds from a plan, at Pass 1's parameters for a 1% error, is +/// built through the executor and fed 10,000 seeded rows: latencies +/// log-uniform over 1 ms to 1 s (declared non-negative), or 1,000 +/// distinct label values or label sets. Its charge +/// (`approx_memory_bytes`; for HydraCms its shared grid over its 10 +/// groups) is compared with `summary_shape` for the same rows. Priced ÷ +/// charged lies within [`BYTES_AGREEMENT_FACTOR`], or the family is in +/// [`KNOWN_BYTES_GAPS`] with its issue and measured ratio. +/// +/// HydraKll is left out: no plan builds it (it has no parameter list, +/// and the executor's factory refuses it). +#[test] +fn priced_bytes_agree_with_the_executor() { + use asap_executor::factory::{create_hydra_cms, create_planner_accumulator}; + use asap_executor::values::Value; + use asap_types::ir::scalar::expr_ir::ColumnRef; + use asap_types::summary_type::{ + ExactKind, ExactParams, HydraParams, SummaryInputExpr, SummaryUpdate, + }; + use SketchAlgorithm as A; + const ROWS: u32 = 10_000; + const DISTINCT_KEYS: u32 = 1_000; + const HYDRA_GROUPS: u32 = 10; + let input_rows = |rows_per_state: u32| StateInputRows { + rows_per_state: Some(u64::from(rows_per_state)), + non_negative: true, + }; + let sketch = FieldDataType::Sketch; + let exact = |kind, params| { + ( + FieldDataType::ExactAggregate(kind, params), + None, + Rows::Values, + ) + }; + // (family, its parameters, the rows it is fed). + let families = [ + ( + sketch(A::Kll), + Some(SketchParams::Kll { k: 269 }), + Rows::Values, + ), + ( + sketch(A::DDSketch), + Some(SketchParams::DDSketch { alpha: 0.01 }), + Rows::Values, + ), + ( + sketch(A::Hll), + Some(SketchParams::Hll { precision: 18 }), + Rows::Values, + ), + ( + sketch(A::Kmv), + Some(SketchParams::Kmv { k: 106_263 }), + Rows::Values, + ), + ( + sketch(A::Cms), + Some(SketchParams::Cms { + width: 272, + depth: 5, + }), + Rows::Keyed, + ), + ( + sketch(A::CountSketch), + Some(SketchParams::CountSketch { + width: 30_000, + depth: 47, + }), + Rows::Keyed, + ), + ( + sketch(A::CmsWithHeap), + Some(SketchParams::CmsWithHeap { + width: 272, + depth: 5, + heap_size: 10, + }), + Rows::Keyed, + ), + ( + sketch(A::CountSketchWithHeap), + Some(SketchParams::CountSketchWithHeap { + width: 30_000, + depth: 47, + heap_size: 10, + }), + Rows::Keyed, + ), + ( + sketch(A::UnivMon), + Some(SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1_024, + layers: 16, + }), + Rows::Labels, + ), + exact(ExactKind::Sum, ExactParams::Sum), + exact(ExactKind::Count, ExactParams::Count), + exact(ExactKind::Min, ExactParams::Min), + exact(ExactKind::Max, ExactParams::Max), + exact(ExactKind::Increase, ExactParams::Increase), + exact(ExactKind::Rate, ExactParams::Rate), + ]; + // Every family the executor declares is checked. + for supported in asap_executor::supported_summary_families() { + assert!( + families + .iter() + .any(|(family, ..)| *family == sketch(supported.summary_type)), + "{:?} is built by the executor but not checked here", + supported.summary_type + ); + } + let value = SummaryUpdate::column(ColumnRef::SampleValue); + let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); + let keyed = SummaryUpdate { + item: item.clone(), + ..value.clone() + }; + let labeled = SummaryUpdate { + item, + weight: SummaryInputExpr::Constant(1.0), + }; + // (family name, priced bytes, charged bytes). + let mut measured: Vec<(String, u64, u64)> = Vec::new(); + for (family, params, rows) in &families { + let input = match rows { + Rows::Values => &value, + Rows::Labels => &labeled, + Rows::Keyed => &keyed, + }; + let mut updater = create_planner_accumulator(family, params.as_ref(), input).unwrap(); + let mut seed = 0x5EED_u64; + for i in 0..ROWS { + let key = (next_unit(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; + match rows { + // One sample a second, its latency log-uniform over 1 ms to 1 s. + Rows::Values => { + let latency_ms = 1_000f64.powf(next_unit(&mut seed)); + updater + .update_single(latency_ms, i64::from(i) * 1_000) + .unwrap(); + } + Rows::Labels => updater + .update_value(&Value::Utf8(label(key).into()), 0) + .unwrap(), + Rows::Keyed => updater.update_keyed(&labels(key), 1.0, 0).unwrap(), + } + } + let charged = updater.into_accumulator().approx_memory_bytes() as u64; + let priced = summary_shape(family, params.as_ref(), input_rows(ROWS)) + .unwrap() + .1; + let name = crate::node::usage::summary_type_name(family); + measured.push((name, priced, charged)); + } + // HydraCms: one grid for every group, priced as one Count-Min per group. + let hydra = SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 64, + }); + let hydra_family = sketch(A::HydraCms); + let mut grid = create_hydra_cms(&hydra_family, Some(&hydra)).unwrap(); + let mut seed = 0x5EED_u64; + for _ in 0..ROWS { + let group = (next_unit(&mut seed) * f64::from(HYDRA_GROUPS)) as u32; + let item = (next_unit(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; + grid.count(&label(group), &label(item), 1).unwrap(); + } + measured.push(( + crate::node::usage::summary_type_name(&hydra_family), + sketch_shape(&hydra, input_rows(ROWS / HYDRA_GROUPS)).1, + (grid.approx_memory_bytes() / HYDRA_GROUPS as usize) as u64, + )); + + let mut failures = Vec::new(); + for (name, priced, charged) in &measured { + let ratio = *priced as f64 / *charged as f64; + let agrees = (1.0 / BYTES_AGREEMENT_FACTOR..=BYTES_AGREEMENT_FACTOR).contains(&ratio); + let line = format!("{name}: {priced} priced, {charged} charged, ratio {ratio:.3}"); + match KNOWN_BYTES_GAPS.iter().find(|(gap, ..)| gap == name) { + Some((_, issue, _)) if agrees => failures.push(format!( + "{line}: agrees now; remove it from KNOWN_BYTES_GAPS (#{issue})" + )), + Some((_, issue, listed)) if (ratio / listed - 1.0).abs() > 0.05 => failures.push( + format!("{line}: listed at {listed} in KNOWN_BYTES_GAPS (#{issue}); update it"), + ), + Some(_) => {} + None if !agrees => failures.push(format!( + "{line}: outside {BYTES_AGREEMENT_FACTOR}x; fix it or list it in KNOWN_BYTES_GAPS with its issue" + )), + None => {} + } + } + for (gap, issue, _) in KNOWN_BYTES_GAPS { + if !measured.iter().any(|(name, ..)| name == gap) { + failures.push(format!( + "KNOWN_BYTES_GAPS lists {gap} (#{issue}), which is not measured" + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index f347c1259..1358e7fcc 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -1,6 +1,11 @@ //! One node of a physical plan DAG: its resource usage and cost //! ([`node_resource_usage_and_cost`], [`NodeCost`]). +// The rules here answer per operator; a wildcard arm would give a new +// operator a default answer instead of failing the build (under +// `-D warnings`). +#![warn(clippy::wildcard_enum_match_arm)] + use std::collections::{BTreeMap, BTreeSet}; use crate::node::analytical::{ @@ -93,14 +98,46 @@ fn state_family<'a>( .map(StateFamily::Sketch) .ok_or(AnalyticalCostError::UnsupportedCandidate)?, FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), - _ => StateFamily::None, + FieldDataType::Plain(_) + | FieldDataType::SummarySample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..) => StateFamily::None, }, Payload::ASAP( ASAPOp::SummaryMerge { .. } | ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }, ) => producer_family.unwrap_or(StateFamily::None), - _ => StateFamily::None, + Payload::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => StateFamily::None, }) } @@ -266,7 +303,7 @@ pub fn node_resource_usage_and_cost<'a>( let speed = settings.resource_speed; let unit_costs = match &settings.cost_model { CostModel::UsageOverTime(unit_cost) => Some(unit_cost), - _ => None, + CostModel::ProvisionedCapacity(_) | CostModel::ClusterCapacity(_) => None, }; let sketch_size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; @@ -312,7 +349,40 @@ pub fn node_resource_usage_and_cost<'a>( }; let sources: BTreeSet<&str> = match &node.payload { Payload::NonASAP(NonASAPOp::Scan { source, .. }) => [source_name(source)].into(), - _ => producers + Payload::NonASAP( + NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => producers .iter() .flat_map(|p| p.sources.iter().copied()) .collect(), @@ -373,7 +443,40 @@ pub fn node_resource_usage_and_cost<'a>( Payload::NonASAP(NonASAPOp::Aggregate { measures, .. }) => { aggregate_lookup(table, node.output_state.timing, measures) } - _ => lookup( + Payload::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => lookup( table, operator, node.output_state.timing, @@ -696,7 +799,20 @@ pub fn node_resource_usage_and_cost<'a>( ), ) } - _ => { + NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. } => { let estimate = estimate_operator( PhysicalOperator::PassThrough, OperatorStatistics::PassThrough { @@ -827,7 +943,11 @@ pub fn node_resource_usage_and_cost<'a>( Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { let per_group = match query { SketchStatistic::TopK { k } => *k as u64, - _ => 1, + SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy + | SketchStatistic::Quantile { .. } + | SketchStatistic::PointCount { .. } + | SketchStatistic::Cardinality => 1, }; let out = edge( input @@ -846,7 +966,14 @@ pub fn node_resource_usage_and_cost<'a>( Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), format!("finalize {} accumulators", input.rows), ), - _ => ( + Payload::ASAP( + ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => ( edge(input.rows)?, Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), format!("{} rows", input.rows), @@ -862,7 +989,39 @@ pub fn node_resource_usage_and_cost<'a>( let units = match &node.payload { Payload::NonASAP(NonASAPOp::Scan { .. }) => out.rows, Payload::ASAP(ASAPOp::SummaryMerge { .. }) => merged_states(producers)?, - _ => input.rows, + Payload::NonASAP( + NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => input.rows, }; let mut detail = detail; // A node with an entry takes its measured time, scanning included; @@ -1106,7 +1265,10 @@ pub fn summary_type_name(summary_type: &FieldDataType) -> String { match summary_type { FieldDataType::Sketch(algorithm) => format!("{algorithm:?}"), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), - other => format!("{other:?}"), + other @ (FieldDataType::Plain(_) + | FieldDataType::SummarySample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..)) => format!("{other:?}"), } } diff --git a/crates/executor/src/summary_kernels/traits.rs b/crates/executor/src/summary_kernels/traits.rs index 372adeac1..0ed24c290 100644 --- a/crates/executor/src/summary_kernels/traits.rs +++ b/crates/executor/src/summary_kernels/traits.rs @@ -8,6 +8,59 @@ pub type KernelError = Box; /// operations physical operators need: merge, typed evaluation and memory /// accounting. Grouping belongs to operators; byte encodings belong to /// `asap_sketchlib` and deployments. +/// +/// A kernel must say what its state holds: [`Self::approx_memory_bytes`] +/// has no default. A kernel that says it builds: +/// +/// ``` +/// use asap_executor::summary_kernels::traits::{AggregateCore, KernelError}; +/// +/// #[derive(Clone)] +/// struct Counter(u64); +/// +/// impl AggregateCore for Counter { +/// fn clone_boxed_core(&self) -> Box { +/// Box::new(self.clone()) +/// } +/// fn as_any(&self) -> &dyn std::any::Any { +/// self +/// } +/// fn as_any_mut(&mut self) -> &mut dyn std::any::Any { +/// self +/// } +/// fn merge_with(&self, _: &dyn AggregateCore) -> Result, KernelError> { +/// Ok(self.clone_boxed_core()) +/// } +/// fn approx_memory_bytes(&self) -> usize { +/// std::mem::size_of::() +/// } +/// } +/// ``` +/// +/// The same kernel without it fails to build (E0046, a missing trait +/// item), instead of being charged a made-up size: +/// +/// ```compile_fail,E0046 +/// use asap_executor::summary_kernels::traits::{AggregateCore, KernelError}; +/// +/// #[derive(Clone)] +/// struct Counter(u64); +/// +/// impl AggregateCore for Counter { +/// fn clone_boxed_core(&self) -> Box { +/// Box::new(self.clone()) +/// } +/// fn as_any(&self) -> &dyn std::any::Any { +/// self +/// } +/// fn as_any_mut(&mut self) -> &mut dyn std::any::Any { +/// self +/// } +/// fn merge_with(&self, _: &dyn AggregateCore) -> Result, KernelError> { +/// Ok(self.clone_boxed_core()) +/// } +/// } +/// ``` pub trait AggregateCore: Send + Sync { fn clone_boxed_core(&self) -> Box; @@ -35,10 +88,9 @@ pub trait AggregateCore: Send + Sync { /// Approximate in-memory footprint, used for execution memory reservations. /// A state shared with other rows (see [`Self::shared_state`]) is not - /// included. - fn approx_memory_bytes(&self) -> usize { - 4096 - } + /// included. No default: each kernel says what it holds, e.g. a KLL + /// state its sketch's `memory_bytes()`. + fn approx_memory_bytes(&self) -> usize; /// The identity and footprint of a state this row shares with other /// rows (one Hydra grid read by every group of a build). Reservations diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 49a361f31..c3da702dd 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -82,7 +82,8 @@ pub enum NonASAPOp> { measures: Vec, /// Output column names, exactly one entry per measure; an empty /// string means the default name (the intent's synthetic name). - #[serde(default)] + /// Required when deserialized: a missing list would not match the + /// measures. output_names: Vec, #[serde(default)] filters: Vec>>, @@ -1446,6 +1447,34 @@ mod tests { } } + /// An `Aggregate` read without `output_names` is rejected instead of + /// getting an empty list that matches none of its measures; with its + /// names it round-trips. E.g. `{"Aggregate": {"measures": ["Rate"], + /// ...}}` with no `"output_names"` fails with a missing field. + #[test] + fn aggregate_output_names_are_required_when_deserialized() { + let aggregate = rate_over(scan( + vec![col("value", DataType::Float64, false)], + None, + vec![], + )); + let json = serde_json::to_value(&aggregate).unwrap(); + let read_back: NonASAPOp = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(read_back, aggregate); + + let mut without_names = json; + without_names["Aggregate"] + .as_object_mut() + .unwrap() + .remove("output_names") + .unwrap(); + let error = serde_json::from_value::(without_names).unwrap_err(); + assert!( + error.to_string().contains("missing field `output_names`"), + "{error}" + ); + } + #[test] fn project_preserves_unique_keys_that_are_passed_through() { let input = scan( diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index a9ab9a4cf..f17d54064 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -23,6 +23,11 @@ //! read through `TimeWindow { ends_before_evaluation_ms: 60_000, //! length_ms: 300_000 }`, `(t - 6m, t - 1m]`, reaching back 360,000 ms. +// The rules here answer per operator; a wildcard arm would give a new +// operator a default answer instead of failing the build (under +// `-D warnings`). +#![warn(clippy::wildcard_enum_match_arm)] + use serde::{Deserialize, Serialize}; use crate::ir::operator::asap::ASAPOp; @@ -128,6 +133,8 @@ impl InputTimeAccess { reads, shifts_back_ms: 0, }; + // Every operator is listed, so a new one fails to build here until + // its input time access is decided. match operator { Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { None => Self { @@ -153,7 +160,42 @@ impl InputTimeAccess { }), ExecutionTiming::QueryTime => reads(InputRows::WhatItsReadersRead), }, - _ => reads(InputRows::CurrentTimeOnly), + // A subquery evaluates its input at other times than its own, + // but is read here as the rows at t. + Operator::NonASAP(NonASAPOp::PromqlSubquery { .. }) => { + reads(InputRows::CurrentTimeOnly) + } + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => reads(InputRows::CurrentTimeOnly), } } } @@ -355,7 +397,39 @@ pub fn windows_computed_from( tumbling_window_width_ms, .. }) => *tumbling_window_width_ms, - _ => { + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => { pending.extend(plan.inputs_of(id)); continue; } @@ -395,6 +469,267 @@ mod tests { assert_eq!(shifted.bounds_ms(1_000_000), (640_000, 940_000)); } + /// Every operator's input time access, written out by hand: a new + /// operator needs a row here, as it needs an arm in + /// [`InputTimeAccess::of`]. E.g. `TimeShift(1h)` at query time reads + /// what its readers read, an hour earlier; `Aggregate` reads the rows at + /// t at either timing. `PromqlSubquery` keeps its current access, the + /// rows at t, although it evaluates its input at other times. + #[test] + fn every_operator_has_its_input_time_access() { + use crate::ir::operator::maintained_population::{ + CurrentSeriesInput, MaintainedPopulation, PopulationInput, PopulationStatistic, + }; + use crate::ir::operator::{ + AggIntent, AtModifier, BinaryOpKind, GroupKeys, JoinKind, Reduction, + RelationalSetOpKind, SampleKind, Source, TimeShift, WindowFuncKind, + }; + use crate::ir::scalar::{ArithmeticOpKind, ColumnRef}; + use crate::ir::schema::{FieldDataType, Schema}; + use crate::ir::{BinaryOperator, Predicate, ScalarExpr, TimeRangeKind}; + use crate::stages::StorageMedium; + use crate::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + use std::time::Duration; + use ExecutionTiming::{IngestionTime, QueryTime}; + use InputRows::*; + + let column = || ScalarExpr::::Column(0); + let predicate = || Predicate(column()); + let non_asap = |op: NonASAPOp| Operator::NonASAP(op); + let asap = |op: ASAPOp| Operator::ASAP(op); + let kll = || FieldDataType::Sketch(SketchAlgorithm::Kll); + let sample_value = || SummaryUpdate::column(ColumnRef::SampleValue); + let one_hour = |at| TimeShift { + offset_ms: 3_600_000, + at, + }; + let shift = |at| { + non_asap(NonASAPOp::TimeShift { + shift: one_hour(at), + child: 0, + }) + }; + let chain = || { + asap(ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: kll(), + input: sample_value(), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }) + }; + // Every operator other than `TimeShift`, `TimeRange` and the chain: + // each reads the rows at t, at either timing. + let rows_at_t: Vec> = vec![ + non_asap(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::default(), + }), + non_asap(NonASAPOp::Values { + rows: vec![], + schema: Schema::default(), + }), + non_asap(NonASAPOp::Filter { + pred: predicate(), + child: 0, + }), + non_asap(NonASAPOp::Project { + cols: vec![], + child: 0, + }), + non_asap(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![String::new()], + filters: vec![], + having: None, + child: 0, + }), + non_asap(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: predicate(), + left: 0, + right: 1, + }), + non_asap(NonASAPOp::SetOp { + kind: RelationalSetOpKind::Union, + all: true, + left: 0, + right: 1, + }), + non_asap(NonASAPOp::Concat { + children: vec![0, 1], + }), + non_asap(NonASAPOp::Dedup { + cols: vec![0], + child: 0, + }), + non_asap(NonASAPOp::Sort { + keys: vec![], + partition_by: GroupKeys::default(), + child: 0, + }), + non_asap(NonASAPOp::Limit { + n: Some(5), + offset: 0, + partition_by: GroupKeys::default(), + child: 0, + }), + non_asap(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), + vector_match: None, + }, + return_bool: false, + lhs: 0, + rhs: 1, + }), + non_asap(NonASAPOp::SQLWindowFunc { + func: WindowFuncKind::RowNumber, + args: vec![], + partition_by: GroupKeys::default(), + order_by: vec![], + frame: None, + output_name: "row".into(), + child: 0, + }), + non_asap(NonASAPOp::PromqlVectorFromScalar(column())), + non_asap(NonASAPOp::PromqlRelabel { + dst: "job".into(), + value: column(), + child: 0, + }), + non_asap(NonASAPOp::PromqlInfoEnrich { + selector: vec![], + child: 0, + }), + non_asap(NonASAPOp::PromqlSeriesSample { + by: GroupKeys::default(), + kind: SampleKind::LimitK(5), + child: 0, + }), + non_asap(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(300), + resolution: None, + child: 0, + }), + asap(ASAPOp::SummaryAgg { + child: 0, + summary_type: kll(), + input: sample_value(), + reduction: Reduction::PerEntity, + filter: None, + }), + asap(ASAPOp::SummaryEstimate { + summary_input: 0, + query: SketchStatistic::Cardinality, + }), + asap(ASAPOp::FinalizeExactAccumulator { child: 0 }), + asap(ASAPOp::MaintainPopulation { + child: 0, + population: MaintainedPopulation { + input: PopulationInput::CurrentSeries(CurrentSeriesInput { + metric: "m".into(), + matchers: vec![], + grouping: vec![], + without: false, + lookback_ms: 300_000, + }), + max_k: 5, + quantiles: false, + }, + }), + asap(ASAPOp::EvaluatePopulation { + child: 0, + evaluation: PopulationStatistic::Sum, + }), + asap(ASAPOp::SummaryMerge { + children: vec![0, 1], + group_by: None, + }), + asap(ASAPOp::SummarySubtract { left: 0, right: 1 }), + asap(ASAPOp::SummaryDelete { + summary_input: 0, + key: 0, + }), + asap(ASAPOp::SummaryJoin { + outer: 0, + inner: 1, + key: 0, + summary_type: kll(), + }), + asap(ASAPOp::Extension { + child: 0, + name: "extension".into(), + }), + ]; + let stored = Lifetime::AcrossEvaluations(StorageMedium::Memory); + let once = Lifetime::OneEvaluation(StorageMedium::Memory); + let access = |reads, shifts_back_ms| InputTimeAccess { + reads, + shifts_back_ms, + }; + let mut table: Vec<(Operator, ExecutionTiming, Lifetime, InputTimeAccess)> = vec![ + ( + shift(None), + QueryTime, + once, + access(WhatItsReadersRead, 3_600_000), + ), + ( + shift(None), + IngestionTime, + once, + access(WhatItsReadersRead, 3_600_000), + ), + ( + shift(None), + IngestionTime, + stored, + access(CurrentTimeOnly, 3_600_000), + ), + ( + shift(Some(AtModifier::End)), + QueryTime, + once, + access(AllRows, 0), + ), + ( + non_asap(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: 0, + }), + QueryTime, + once, + access(LastWindow { length_ms: 300_000 }, 0), + ), + (chain(), QueryTime, once, access(WhatItsReadersRead, 0)), + ( + chain(), + IngestionTime, + stored, + access(LastWindow { length_ms: 60_000 }, 0), + ), + ]; + for operator in rows_at_t { + for timing in [QueryTime, IngestionTime] { + table.push((operator.clone(), timing, stored, access(CurrentTimeOnly, 0))); + } + } + for (operator, timing, lifetime, expected) in table { + assert_eq!( + InputTimeAccess::of(&operator, timing, lifetime), + expected, + "{operator:?} at {timing:?}, {lifetime:?}" + ); + } + } + /// A hand-built plan: each node's readers and input time access, and /// whether node 0 is raw data. The walks up read only these. struct Plan { From e4605c18ff1097a63ff869270a22603ce0874ae1 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:03:15 +0000 Subject: [PATCH 2/4] refactor: answer window questions once on Operator; review fixes - Operator::chain_tumbling_window_width_ms, covered_window_ms and keeps_input_time, each listing every operator, replace the five duplicated variant lists in time_window and node::usage; node.rs is linted, with a table test per method. - CostModel::unit_costs, used by node usage, plan_cost, the request overhead and the raw-data disk cost. - edge_roles and check_raw_data_windows list every operator; the raw-data reader check uses covered_window_ms. - Lint comments name the identity checks kept as matches!/if let; add match_wildcard_for_single_variants; summary_type_name keeps its Debug fallback with an allow and a reason. - Bytes test: known gaps on each family's row (KnownBytesGap, keyed by FieldDataType), the exact-accumulator gap under its own issue, Rows::update shared by both tests, BYTES_AGREEMENT_FACTOR everywhere, next_uniform, and per-row-kind counts (10,000 values, 1,000 label rows): about 1.8 s instead of 15-30 s in debug, same ratios. - Time-access test trimmed to the operators with a decision; the two AggregateCore doctests dropped. --- crates/cost/Cargo.toml | 3 +- crates/cost/src/model.rs | 13 + crates/cost/src/node/state_bytes/tests.rs | 228 +++++++------- crates/cost/src/node/usage/mod.rs | 36 ++- crates/cost/src/plan/cost.rs | 17 +- crates/executor/src/summary_kernels/traits.rs | 53 ---- crates/types/src/ir/operator/node.rs | 223 +++++++++++++- .../src/plan/materialized_view/raw_data.rs | 40 ++- .../types/src/plan/materialized_view/views.rs | 11 +- crates/types/src/plan/physical_asap_dag.rs | 27 +- crates/types/src/plan/time_window.rs | 282 +++--------------- 11 files changed, 499 insertions(+), 434 deletions(-) diff --git a/crates/cost/Cargo.toml b/crates/cost/Cargo.toml index 5bbdc691c..e0298ba38 100644 --- a/crates/cost/Cargo.toml +++ b/crates/cost/Cargo.toml @@ -16,5 +16,6 @@ serde_json = "1" [dev-dependencies] # The executor's charge for each summary state, checked against the bytes -# the state byte models give for it plus the accumulator's own fields (#947). +# the state byte models give for it plus the accumulator's own fields, and +# against priced bytes on seeded workload data (#947). asap-executor = { path = "../executor" } diff --git a/crates/cost/src/model.rs b/crates/cost/src/model.rs index caba6b63b..a85f7874b 100644 --- a/crates/cost/src/model.rs +++ b/crates/cost/src/model.rs @@ -155,6 +155,19 @@ pub enum CostModel { ClusterCapacity(ClusterLimits), } +impl CostModel { + /// The per-unit costs, which only cost model 1 prices each node's + /// usage with: `UsageOverTime(u)` gives `Some(&u)`; cost model 2 prices + /// the plan's peaks, not each node, and cost model 3 has no prices, so + /// both give `None`. + pub fn unit_costs(&self) -> Option<&ResourceUnitCost> { + match self { + CostModel::UsageOverTime(unit_cost) => Some(unit_cost), + CostModel::ProvisionedCapacity(_) | CostModel::ClusterCapacity(_) => None, + } + } +} + /// A fixed cluster's capacity (cost model 3) and how to choose within it. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index 17274e2fb..fae2379e3 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -34,6 +34,42 @@ enum Rows { Keyed, } +impl Rows { + /// How many rows [`priced_bytes_agree_with_the_executor`] feeds a + /// family of these rows, priced for the same count. 10,000 values: a + /// KLL state's charge reaches its priced size only past about 3,000 + /// rows. 1,000 label rows: the label families' grids have a fixed + /// size, so their ratio is the same at any count, and 1,000 rows over + /// 1,000 keys still churn a heap of 10; at 10,000 rows Count Sketch + /// with a heap alone took about 22 s in a debug build. + fn fed(&self) -> u32 { + match self { + Rows::Values => 10_000, + Rows::Labels | Rows::Keyed => 1_000, + } + } + + /// The update a family fed these rows reads: the sample value; an + /// `item` label counted once a row; or an `item` label set weighted by + /// the sample value. + fn update(&self) -> asap_types::summary_type::SummaryUpdate { + use asap_types::ir::scalar::expr_ir::ColumnRef; + use asap_types::summary_type::{SummaryInputExpr, SummaryUpdate}; + let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); + match self { + Rows::Values => SummaryUpdate::column(ColumnRef::SampleValue), + Rows::Labels => SummaryUpdate { + item, + weight: SummaryInputExpr::Constant(1.0), + }, + Rows::Keyed => SummaryUpdate { + item, + ..SummaryUpdate::column(ColumnRef::SampleValue) + }, + } + } +} + /// A 16-byte label value, `i` zero-padded: its key has /// [`ASSUMED_VALUE_KEY_BYTES`]. fn label(i: u32) -> String { @@ -169,7 +205,8 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { /// family. It equals the priced bytes where `asap_sketchlib`'s estimate /// is exact (Count-Min, Count Sketch, HLL, KMV at `n` distinct items) and /// for KLL up to `k` rows. For DDSketch, the heap sketches and UnivMon -/// the price bounds it, by at most 1.5 times once the state is full; +/// the price bounds it, by at most [`BYTES_AGREEMENT_FACTOR`] times once +/// the state is full; /// UnivMon's price also bounds the bytes its sketch holds. /// KLL past `k` rows is checked only loosely: the planner's formula /// leaves out the compaction scratch space, a known gap until a @@ -185,8 +222,6 @@ fn state_bytes_bound_the_executors_charge() { use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; use asap_executor::summary_kernels::HllSketchAccumulator; use asap_executor::values::Value; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::summary_type::{SummaryInputExpr, SummaryUpdate}; use std::mem::{align_of, size_of}; use SketchAlgorithm as A; /// How the priced bytes relate to the sketch's. @@ -197,7 +232,8 @@ fn state_bytes_bound_the_executors_charge() { Kll { k: u32, }, - /// At least, and at most 1.5 times once the state is full. + /// At least, and at most [`BYTES_AGREEMENT_FACTOR`] times once the + /// state is full. Bound, } // (family, parameters, rows fed, how it is priced, the @@ -308,16 +344,6 @@ fn state_bytes_bound_the_executors_charge() { 4_096, ), ]; - let value = SummaryUpdate::column(ColumnRef::SampleValue); - let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); - let keyed = SummaryUpdate { - item: item.clone(), - ..value.clone() - }; - let labeled = SummaryUpdate { - item, - weight: SummaryInputExpr::Constant(1.0), - }; assert_eq!( Value::Utf8(label(0).into()).key().unwrap().len(), ASSUMED_VALUE_KEY_BYTES @@ -332,12 +358,8 @@ fn state_bytes_bound_the_executors_charge() { for (algorithm, params, rows, priced_as, fields, full) in &families { for n in [1, 100, *full] { let family = FieldDataType::Sketch(*algorithm); - let input = match rows { - Rows::Values => &value, - Rows::Labels => &labeled, - Rows::Keyed => &keyed, - }; - let mut updater = create_planner_accumulator(&family, Some(params), input).unwrap(); + let mut updater = + create_planner_accumulator(&family, Some(params), &rows.update()).unwrap(); for i in 0..n { match rows { // Distinct magnitudes over the assumed range, in a @@ -379,7 +401,7 @@ fn state_bytes_bound_the_executors_charge() { assert!(charged <= priced, "{context}, {charged} charged"); if n == *full { assert!( - priced as f64 <= 1.5 * charged as f64, + priced as f64 <= BYTES_AGREEMENT_FACTOR * charged as f64, "{context}, {charged} charged" ); } @@ -390,7 +412,8 @@ fn state_bytes_bound_the_executors_charge() { // The estimate is a heap whose digest index has grown once // more under churn, which the test's labels do not reach: a // full heap holds about two thirds of it (1.52 times less - // here), so this bound is 1.6 times rather than 1.5. + // here), so this bound is 1.6 times rather than + // `BYTES_AGREEMENT_FACTOR`. if let Some(univmon) = state.as_any().downcast_ref::() { let held = fields + univmon.sketch().memory_bytes(); assert!(held <= charged, "{context}, {held} held"); @@ -403,36 +426,25 @@ fn state_bytes_bound_the_executors_charge() { } /// How far plan selection's bytes for a state may be from what the -/// executor charges for it in [`priced_bytes_agree_with_the_executor`]: -/// priced ÷ charged in `[1 / 1.5, 1.5]`. +/// executor charges for it: priced ÷ charged in `[1 / 1.5, 1.5]` on +/// seeded data ([`priced_bytes_agree_with_the_executor`]), and priced at +/// most 1.5 times the charge once a worst-case state is full +/// ([`state_bytes_bound_the_executors_charge`]). const BYTES_AGREEMENT_FACTOR: f64 = 1.5; -/// The families whose priced bytes are known to disagree with the -/// executor's beyond [`BYTES_AGREEMENT_FACTOR`]: (family, the issue that -/// fixes it, priced ÷ charged as measured by -/// [`priced_bytes_agree_with_the_executor`]). A family leaves the list -/// when its issue is fixed: the test fails while it is listed but agrees, -/// or when its ratio moves off the one listed. -const KNOWN_BYTES_GAPS: &[(&str, u32, f64)] = &[ - // Priced for values over nine decades (`DDSKETCH_ASSUMED_VALUE_RANGE`); - // these span three. - ("DDSketch", 1313, 2.982), - // Priced as one element of the executor's state column (56 bytes); - // a boxed accumulator charges its whole struct (136 bytes). - ("exact Sum accumulator", 1301, 0.412), - ("exact Count accumulator", 1301, 0.412), - ("exact Min accumulator", 1301, 0.412), - ("exact Max accumulator", 1301, 0.412), - ("exact Increase accumulator", 1301, 0.412), - ("exact Rate accumulator", 1301, 0.412), - // Priced as one Count-Min per group; the executor keeps one grid for - // all of them, here 3 × 64 cells over 10 groups. - ("HydraCms", 947, 0.104), -]; +/// A family's priced bytes known to disagree with the executor's beyond +/// [`BYTES_AGREEMENT_FACTOR`], on its row of +/// [`priced_bytes_agree_with_the_executor`]: the issue that fixes it and +/// priced ÷ charged as measured there. The test fails when the family +/// agrees (remove the gap) or its ratio moves more than 5% off this one. +struct KnownBytesGap { + issue: u32, + priced_over_charged: f64, +} /// The next value of the seeded stream `state` (splitmix64), uniform in /// `[0, 1)`, so the rows every family is fed are fixed. -fn next_unit(state: &mut u64) -> f64 { +fn next_uniform(state: &mut u64) -> f64 { *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); let mut z = *state; z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); @@ -444,13 +456,14 @@ fn next_unit(state: &mut u64) -> f64 { /// on data shaped like a workload's, not only on the worst case /// [`state_bytes_bound_the_executors_charge`] feeds. Every family the /// executor builds from a plan, at Pass 1's parameters for a 1% error, is -/// built through the executor and fed 10,000 seeded rows: latencies -/// log-uniform over 1 ms to 1 s (declared non-negative), or 1,000 -/// distinct label values or label sets. Its charge +/// built through the executor and fed seeded rows ([`Rows::fed`]): +/// 10,000 latencies log-uniform over 1 ms to 1 s (declared +/// non-negative), or 1,000 rows of 1,000 distinct label values or label +/// sets. Its charge /// (`approx_memory_bytes`; for HydraCms its shared grid over its 10 /// groups) is compared with `summary_shape` for the same rows. Priced ÷ -/// charged lies within [`BYTES_AGREEMENT_FACTOR`], or the family is in -/// [`KNOWN_BYTES_GAPS`] with its issue and measured ratio. +/// charged lies within [`BYTES_AGREEMENT_FACTOR`], or the family's row +/// carries its [`KnownBytesGap`]. /// /// HydraKll is left out: no plan builds it (it has no parameter list, /// and the executor's factory refuses it). @@ -458,47 +471,58 @@ fn next_unit(state: &mut u64) -> f64 { fn priced_bytes_agree_with_the_executor() { use asap_executor::factory::{create_hydra_cms, create_planner_accumulator}; use asap_executor::values::Value; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::summary_type::{ - ExactKind, ExactParams, HydraParams, SummaryInputExpr, SummaryUpdate, - }; + use asap_types::summary_type::{ExactKind, ExactParams, HydraParams}; use SketchAlgorithm as A; - const ROWS: u32 = 10_000; const DISTINCT_KEYS: u32 = 1_000; const HYDRA_GROUPS: u32 = 10; let input_rows = |rows_per_state: u32| StateInputRows { rows_per_state: Some(u64::from(rows_per_state)), non_negative: true, }; + let gap = |issue, priced_over_charged| { + Some(KnownBytesGap { + issue, + priced_over_charged, + }) + }; let sketch = FieldDataType::Sketch; + // Priced as one 56-byte record of the executor's exact state column; + // the accumulator object built here charges its whole 136-byte struct. let exact = |kind, params| { ( FieldDataType::ExactAggregate(kind, params), None, Rows::Values, + gap(1333, 0.412), ) }; - // (family, its parameters, the rows it is fed). + // (family, its parameters, the rows it is fed, its known gap). let families = [ ( sketch(A::Kll), Some(SketchParams::Kll { k: 269 }), Rows::Values, + None, ), + // Priced for values over nine decades + // (`DDSKETCH_ASSUMED_VALUE_RANGE`); these span three. ( sketch(A::DDSketch), Some(SketchParams::DDSketch { alpha: 0.01 }), Rows::Values, + gap(1313, 2.982), ), ( sketch(A::Hll), Some(SketchParams::Hll { precision: 18 }), Rows::Values, + None, ), ( sketch(A::Kmv), Some(SketchParams::Kmv { k: 106_263 }), Rows::Values, + None, ), ( sketch(A::Cms), @@ -507,6 +531,7 @@ fn priced_bytes_agree_with_the_executor() { depth: 5, }), Rows::Keyed, + None, ), ( sketch(A::CountSketch), @@ -515,6 +540,7 @@ fn priced_bytes_agree_with_the_executor() { depth: 47, }), Rows::Keyed, + None, ), ( sketch(A::CmsWithHeap), @@ -524,6 +550,7 @@ fn priced_bytes_agree_with_the_executor() { heap_size: 10, }), Rows::Keyed, + None, ), ( sketch(A::CountSketchWithHeap), @@ -533,6 +560,7 @@ fn priced_bytes_agree_with_the_executor() { heap_size: 10, }), Rows::Keyed, + None, ), ( sketch(A::UnivMon), @@ -543,6 +571,7 @@ fn priced_bytes_agree_with_the_executor() { layers: 16, }), Rows::Labels, + None, ), exact(ExactKind::Sum, ExactParams::Sum), exact(ExactKind::Count, ExactParams::Count), @@ -561,32 +590,19 @@ fn priced_bytes_agree_with_the_executor() { supported.summary_type ); } - let value = SummaryUpdate::column(ColumnRef::SampleValue); - let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); - let keyed = SummaryUpdate { - item: item.clone(), - ..value.clone() - }; - let labeled = SummaryUpdate { - item, - weight: SummaryInputExpr::Constant(1.0), - }; - // (family name, priced bytes, charged bytes). - let mut measured: Vec<(String, u64, u64)> = Vec::new(); - for (family, params, rows) in &families { - let input = match rows { - Rows::Values => &value, - Rows::Labels => &labeled, - Rows::Keyed => &keyed, - }; - let mut updater = create_planner_accumulator(family, params.as_ref(), input).unwrap(); + // (family, priced bytes, charged bytes, its known gap). + let mut measured = Vec::new(); + for (family, params, rows, known_gap) in families { + let rows_fed = rows.fed(); + let mut updater = + create_planner_accumulator(&family, params.as_ref(), &rows.update()).unwrap(); let mut seed = 0x5EED_u64; - for i in 0..ROWS { - let key = (next_unit(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; + for i in 0..rows_fed { + let key = (next_uniform(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; match rows { // One sample a second, its latency log-uniform over 1 ms to 1 s. Rows::Values => { - let latency_ms = 1_000f64.powf(next_unit(&mut seed)); + let latency_ms = 1_000f64.powf(next_uniform(&mut seed)); updater .update_single(latency_ms, i64::from(i) * 1_000) .unwrap(); @@ -598,13 +614,14 @@ fn priced_bytes_agree_with_the_executor() { } } let charged = updater.into_accumulator().approx_memory_bytes() as u64; - let priced = summary_shape(family, params.as_ref(), input_rows(ROWS)) + let priced = summary_shape(&family, params.as_ref(), input_rows(rows_fed)) .unwrap() .1; - let name = crate::node::usage::summary_type_name(family); - measured.push((name, priced, charged)); + measured.push((family, priced, charged, known_gap)); } - // HydraCms: one grid for every group, priced as one Count-Min per group. + // HydraCms: one grid for every group, priced as one Count-Min per + // group; the executor keeps one grid for all of them, here 3 × 64 + // cells over 10 groups. let hydra = SketchParams::Hydra(HydraParams::HydraCms { width: 272, depth: 5, @@ -613,43 +630,44 @@ fn priced_bytes_agree_with_the_executor() { }); let hydra_family = sketch(A::HydraCms); let mut grid = create_hydra_cms(&hydra_family, Some(&hydra)).unwrap(); + let hydra_rows_fed = Rows::Keyed.fed(); let mut seed = 0x5EED_u64; - for _ in 0..ROWS { - let group = (next_unit(&mut seed) * f64::from(HYDRA_GROUPS)) as u32; - let item = (next_unit(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; + for _ in 0..hydra_rows_fed { + let group = (next_uniform(&mut seed) * f64::from(HYDRA_GROUPS)) as u32; + let item = (next_uniform(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; grid.count(&label(group), &label(item), 1).unwrap(); } measured.push(( - crate::node::usage::summary_type_name(&hydra_family), - sketch_shape(&hydra, input_rows(ROWS / HYDRA_GROUPS)).1, + hydra_family, + sketch_shape(&hydra, input_rows(hydra_rows_fed / HYDRA_GROUPS)).1, (grid.approx_memory_bytes() / HYDRA_GROUPS as usize) as u64, + gap(947, 0.104), )); let mut failures = Vec::new(); - for (name, priced, charged) in &measured { + for (family, priced, charged, known_gap) in &measured { let ratio = *priced as f64 / *charged as f64; let agrees = (1.0 / BYTES_AGREEMENT_FACTOR..=BYTES_AGREEMENT_FACTOR).contains(&ratio); - let line = format!("{name}: {priced} priced, {charged} charged, ratio {ratio:.3}"); - match KNOWN_BYTES_GAPS.iter().find(|(gap, ..)| gap == name) { - Some((_, issue, _)) if agrees => failures.push(format!( - "{line}: agrees now; remove it from KNOWN_BYTES_GAPS (#{issue})" + let line = format!( + "{}: {priced} priced, {charged} charged, ratio {ratio:.3}", + crate::node::usage::summary_type_name(family) + ); + match known_gap { + Some(KnownBytesGap { issue, .. }) if agrees => { + failures.push(format!("{line}: agrees now; remove its known gap (#{issue})")) + } + Some(KnownBytesGap { + issue, + priced_over_charged, + }) if (ratio / priced_over_charged - 1.0).abs() > 0.05 => failures.push(format!( + "{line}: its known gap (#{issue}) is {priced_over_charged}; update it" )), - Some((_, issue, listed)) if (ratio / listed - 1.0).abs() > 0.05 => failures.push( - format!("{line}: listed at {listed} in KNOWN_BYTES_GAPS (#{issue}); update it"), - ), Some(_) => {} None if !agrees => failures.push(format!( - "{line}: outside {BYTES_AGREEMENT_FACTOR}x; fix it or list it in KNOWN_BYTES_GAPS with its issue" + "{line}: outside {BYTES_AGREEMENT_FACTOR}x; fix it or give its row a known gap with its issue" )), None => {} } } - for (gap, issue, _) in KNOWN_BYTES_GAPS { - if !measured.iter().any(|(name, ..)| name == gap) { - failures.push(format!( - "KNOWN_BYTES_GAPS lists {gap} (#{issue}), which is not measured" - )); - } - } assert!(failures.is_empty(), "{}", failures.join("\n")); } diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 1358e7fcc..66e2d1848 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -1,10 +1,19 @@ //! One node of a physical plan DAG: its resource usage and cost //! ([`node_resource_usage_and_cost`], [`NodeCost`]). -// The rules here answer per operator; a wildcard arm would give a new -// operator a default answer instead of failing the build (under -// `-D warnings`). -#![warn(clippy::wildcard_enum_match_arm)] +// The rules here answer per operator, summary type or cost model, and +// each lists every variant: a wildcard arm would give a new variant a +// default answer instead of failing the build (under `-D warnings`). +// Clippy does not check `matches!` and `if let`, so they stay only for a +// check of one variant's identity, where every other variant's answer is +// "no" by definition: "is it a `Scan`" (a scan's retained raw bytes), and +// `raw_data_samples_non_negative`'s "is its producer a `Scan` or a +// `TimeRange`" (raw samples; any other producer's are not, the safe +// answer). +#![warn( + clippy::wildcard_enum_match_arm, + clippy::match_wildcard_for_single_variants +)] use std::collections::{BTreeMap, BTreeSet}; @@ -19,8 +28,8 @@ use crate::node::state_bytes::{ }; use crate::{aggregate_lookup, lookup, StateFamily}; use crate::{ - CalibrationFit, CostInputs, CostModel, NodeCostSource, OperatorKind, COST_SOURCE, - MEASURED_COST_SOURCE, MIXED_COST_SOURCE, + CalibrationFit, CostInputs, NodeCostSource, OperatorKind, COST_SOURCE, MEASURED_COST_SOURCE, + MIXED_COST_SOURCE, }; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; use asap_types::ir::properties::ExecutionTiming; @@ -301,10 +310,7 @@ pub fn node_resource_usage_and_cost<'a>( let formula_scale = settings.frozen_calibration_factor.unwrap_or(1.0); let table = &settings.measured_operation_costs; let speed = settings.resource_speed; - let unit_costs = match &settings.cost_model { - CostModel::UsageOverTime(unit_cost) => Some(unit_cost), - CostModel::ProvisionedCapacity(_) | CostModel::ClusterCapacity(_) => None, - }; + let unit_costs = settings.cost_model.unit_costs(); let sketch_size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; let mut fit = CalibrationFit::default(); @@ -633,7 +639,7 @@ pub fn node_resource_usage_and_cost<'a>( let group_count = groups(reduction)?; let keys = match reduction { Reduction::Reduce(keys) => keys.keys().len() as u64, - _ => 1, + Reduction::PerEntity => 1, }; let out = edge(group_count)?; let held = input.rows.div_ceil(group_count.max(1)); @@ -1261,14 +1267,14 @@ fn merged_states(producers: &[ProducerOutput<'_>]) -> Result String { match summary_type { FieldDataType::Sketch(algorithm) => format!("{algorithm:?}"), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), - other @ (FieldDataType::Plain(_) - | FieldDataType::SummarySample(..) - | FieldDataType::Wavelet(..) - | FieldDataType::StatModel(..)) => format!("{other:?}"), + other => format!("{other:?}"), } } diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 47922c71c..34e563225 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -97,10 +97,7 @@ pub fn plan_cost( let mut fit = CalibrationFit::default(); let speed = settings.resource_speed; // Cost model 1 costs each node as it is measured. - let unit_costs = match &settings.cost_model { - CostModel::UsageOverTime(unit_cost) => Some(unit_cost), - _ => None, - }; + let unit_costs = settings.cost_model.unit_costs(); // Each node's `(multiplier, per_second)` ([`computations`]), then its // usage and cost ([`node_resource_usage_and_cost`]), children first. let multipliers = computations(dag, entries, settings.non_repeating_query_horizon_seconds) @@ -403,7 +400,7 @@ pub fn request_overhead_cost_per_second( entries: &[QueryWorkloadEntry], settings: &CostInputs, ) -> Result { - let CostModel::UsageOverTime(unit_cost) = &settings.cost_model else { + let Some(unit_cost) = settings.cost_model.unit_costs() else { return Ok(0.0); }; if settings.measured_operation_costs.ns_per_evaluation <= 0.0 { @@ -423,12 +420,10 @@ pub fn request_overhead_cost_per_second( /// `None` under cost models 2 and 3. A plan keeps each source once, for its /// scan that keeps the most ([`plan_cost`]). pub fn raw_data_disk_cost_per_second(bytes: u64, settings: &CostInputs) -> Option { - match &settings.cost_model { - CostModel::UsageOverTime(unit_cost) => { - Some(unit_cost.cost_per_disk_byte_second * bytes as f64) - } - _ => None, - } + settings + .cost_model + .unit_costs() + .map(|unit_cost| unit_cost.cost_per_disk_byte_second * bytes as f64) } /// The window a query-time `Scan` `node` of `dag` reads per execution: the diff --git a/crates/executor/src/summary_kernels/traits.rs b/crates/executor/src/summary_kernels/traits.rs index 0ed24c290..b7aa08e57 100644 --- a/crates/executor/src/summary_kernels/traits.rs +++ b/crates/executor/src/summary_kernels/traits.rs @@ -8,59 +8,6 @@ pub type KernelError = Box; /// operations physical operators need: merge, typed evaluation and memory /// accounting. Grouping belongs to operators; byte encodings belong to /// `asap_sketchlib` and deployments. -/// -/// A kernel must say what its state holds: [`Self::approx_memory_bytes`] -/// has no default. A kernel that says it builds: -/// -/// ``` -/// use asap_executor::summary_kernels::traits::{AggregateCore, KernelError}; -/// -/// #[derive(Clone)] -/// struct Counter(u64); -/// -/// impl AggregateCore for Counter { -/// fn clone_boxed_core(&self) -> Box { -/// Box::new(self.clone()) -/// } -/// fn as_any(&self) -> &dyn std::any::Any { -/// self -/// } -/// fn as_any_mut(&mut self) -> &mut dyn std::any::Any { -/// self -/// } -/// fn merge_with(&self, _: &dyn AggregateCore) -> Result, KernelError> { -/// Ok(self.clone_boxed_core()) -/// } -/// fn approx_memory_bytes(&self) -> usize { -/// std::mem::size_of::() -/// } -/// } -/// ``` -/// -/// The same kernel without it fails to build (E0046, a missing trait -/// item), instead of being charged a made-up size: -/// -/// ```compile_fail,E0046 -/// use asap_executor::summary_kernels::traits::{AggregateCore, KernelError}; -/// -/// #[derive(Clone)] -/// struct Counter(u64); -/// -/// impl AggregateCore for Counter { -/// fn clone_boxed_core(&self) -> Box { -/// Box::new(self.clone()) -/// } -/// fn as_any(&self) -> &dyn std::any::Any { -/// self -/// } -/// fn as_any_mut(&mut self) -> &mut dyn std::any::Any { -/// self -/// } -/// fn merge_with(&self, _: &dyn AggregateCore) -> Result, KernelError> { -/// Ok(self.clone_boxed_core()) -/// } -/// } -/// ``` pub trait AggregateCore: Send + Sync { fn clone_boxed_core(&self) -> Box; diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 7fa100936..a0e0da6bb 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -5,6 +5,17 @@ //! each physical candidate holds its own //! ([`crate::stages::PhysicalCandidate::properties`]). +// The questions answered here per operator (e.g. the window it covers) +// list every operator: a wildcard arm would give a new operator a default +// answer instead of failing the build (under `-D warnings`). Clippy does +// not check `if let` and `matches!`, so they stay only for a check of one +// variant's identity, where every other operator's answer is "no" by +// definition: "is it a `Scan`" (only a scan reads a source itself). +#![warn( + clippy::wildcard_enum_match_arm, + clippy::match_wildcard_for_single_variants +)] + use std::collections::{BTreeSet, HashSet}; use std::rc::Rc; @@ -108,6 +119,58 @@ impl Operator { } } + /// The window of input rows one output covers, in ms: a `TimeRange`'s + /// range or a chain's tumbling window; `None` for every other operator. + /// E.g. `TimeRange(5m)`: `Some(300_000)`; a chain of 1-minute windows: + /// `Some(60_000)`; `Filter`: `None`. A `PromqlSubquery` evaluates its + /// input over a range, but each of its outputs is one evaluation, so it + /// covers none here. + pub fn covered_window_ms(&self) -> Option { + match self { + // Saturates, so `Some` means exactly "a window", whatever its length. + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => { + Some(u64::try_from(range.as_millis()).unwrap_or(u64::MAX)) + } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => None, + } + } + /// The anchor of a `TimeShift` with `@ end()` or `@ start()`, whose /// input runs once per request; `None` for any other operator (`@ ` /// included). @@ -209,14 +272,13 @@ impl Operator { /// own, else the union over its inputs' /// ([`OperatorInput::scanned_sources`]). pub(crate) fn scanned_sources(&self) -> BTreeSet { - match self { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), - operator => operator - .children() - .into_iter() - .flat_map(OperatorInput::scanned_sources) - .collect(), + if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = self { + return BTreeSet::from([source.clone()]); } + self.children() + .into_iter() + .flat_map(OperatorInput::scanned_sources) + .collect() } pub(crate) fn output_schema(&self) -> Result { @@ -440,3 +502,150 @@ impl OperatorNode { Ok(()) } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::ir::operator::{AggIntent, Reduction, TimeShift}; + use crate::ir::scalar::ColumnRef; + use crate::ir::schema::FieldDataType; + use crate::ir::{Predicate, ScalarExpr, TimeRangeKind}; + use crate::summary_type::{SketchAlgorithm, SummaryUpdate}; + + fn scan() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::default(), + }) + } + + fn filter() -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Column(0)), + child: 0, + }) + } + + fn project() -> Operator { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![], + child: 0, + }) + } + + fn time_shift_one_hour() -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: 0, + }) + } + + fn aggregate() -> Operator { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![String::new()], + filters: vec![], + having: None, + child: 0, + }) + } + + fn time_range_five_minutes() -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: 0, + }) + } + + fn subquery_five_minutes() -> Operator { + Operator::NonASAP(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(300), + resolution: None, + child: 0, + }) + } + + fn summary_agg() -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) + } + + fn chain_one_minute() -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }) + } + + /// Only a chain has a tumbling-window width; a `TimeRange` covers a + /// window but tumbles none. + #[test] + fn only_a_chain_has_a_tumbling_window_width() { + let table = [ + (chain_one_minute(), Some(60_000)), + (time_range_five_minutes(), None), + (summary_agg(), None), + (filter(), None), + ]; + for (operator, expected) in table { + assert_eq!( + operator.chain_tumbling_window_width_ms(), + expected, + "{operator:?}" + ); + } + } + + /// A `TimeRange` covers its range and a chain its tumbling window; a + /// subquery's range is not a covered window, as each of its outputs is + /// one evaluation. + #[test] + fn a_time_range_and_a_chain_cover_a_window() { + let table = [ + (time_range_five_minutes(), Some(300_000)), + (chain_one_minute(), Some(60_000)), + (subquery_five_minutes(), None), + (time_shift_one_hour(), None), + (aggregate(), None), + (scan(), None), + ]; + for (operator, expected) in table { + assert_eq!(operator.covered_window_ms(), expected, "{operator:?}"); + } + } + + /// `Filter`, `Project` and `TimeShift` keep their input's time window; + /// an operator that combines rows or covers its own window does not. + #[test] + fn filters_projections_and_shifts_keep_input_time() { + let table = [ + (filter(), true), + (project(), true), + (time_shift_one_hour(), true), + (aggregate(), false), + (time_range_five_minutes(), false), + (chain_one_minute(), false), + (scan(), false), + ]; + for (operator, expected) in table { + assert_eq!(operator.keeps_input_window(), expected, "{operator:?}"); + } + } +} diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index 707899c13..676e087ac 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -117,7 +117,10 @@ impl PhysicalASAPDAGNode { } impl ViewIndex<'_> { - /// See [`PhysicalASAPDAG::check_raw_data_windows`]. + /// See [`PhysicalASAPDAG::check_raw_data_windows`]. Every operator is + /// listed: a new one fails to build here until it is decided whether + /// it holds a window over raw data to check. + #[warn(clippy::wildcard_enum_match_arm)] pub(super) fn check_raw_data_windows(&self) -> Result<(), InstallableDagError> { for node in &self.dag.nodes { if !self.reading_raw_data.contains(&node.id) { @@ -130,7 +133,40 @@ impl ViewIndex<'_> { Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { self.check_raw_window_below(node.id)?; } - _ => {} + // No window of its own over the raw data. + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => {} } } Ok(()) diff --git a/crates/types/src/plan/materialized_view/views.rs b/crates/types/src/plan/materialized_view/views.rs index 835abdc39..5348bf15b 100644 --- a/crates/types/src/plan/materialized_view/views.rs +++ b/crates/types/src/plan/materialized_view/views.rs @@ -200,11 +200,12 @@ impl<'a> ViewIndex<'a> { .flatten() .any(|input| match payload(input) { Some(Operator::NonASAP(NonASAPOp::Scan { .. })) => true, - Some( - Operator::NonASAP(NonASAPOp::TimeRange { .. }) - | Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ) => false, - _ => reading_raw_data.contains(input), + // A window (`TimeRange`, chain) between it and the raw + // data: it reads the window's output, not raw rows. + Some(operator) if operator.covered_window_ms().is_some() => false, + // Any other operator passes on whether its input reads + // raw data. + Some(_) | None => reading_raw_data.contains(input), }); if reads { reading_raw_data.insert(*id); diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 7e91ce4b5..afc2ae94b 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -567,6 +567,10 @@ impl PhysicalASAPDAG { /// A node referenced several times from the scalar expressions is one /// `ScalarRef` input, as cost lowering reads it once; ids are per node /// pointer, so this dedupes by pointer. +/// +/// Every operator is listed: a new one fails to build here until its input +/// roles are decided, rather than getting one `Input`. +#[warn(clippy::wildcard_enum_match_arm)] fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { use EdgeRole::*; let inputs: Vec = match operator { @@ -578,12 +582,31 @@ fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> NonASAPOp::Scan { .. } | NonASAPOp::Values { .. } | NonASAPOp::PromqlVectorFromScalar(_) => vec![], - _ => vec![Input], + NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. } => vec![Input], }, Operator::ASAP(op) => match op { ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], - _ => vec![Input], + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::Extension { .. } => vec![Input], }, }; let children = operator.children(); diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index f17d54064..a2676b021 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -23,10 +23,17 @@ //! read through `TimeWindow { ends_before_evaluation_ms: 60_000, //! length_ms: 300_000 }`, `(t - 6m, t - 1m]`, reaching back 360,000 ms. -// The rules here answer per operator; a wildcard arm would give a new -// operator a default answer instead of failing the build (under -// `-D warnings`). -#![warn(clippy::wildcard_enum_match_arm)] +// The rules here answer per operator, and each lists every operator: a +// wildcard arm would give a new operator a default answer instead of +// failing the build (under `-D warnings`). Clippy does not check +// `matches!` and `let ... else`, so they stay only for a check of one +// variant's identity, where every other operator's answer is "no" by +// definition: "is it a `Scan`" (only a raw scan's read at t needs every +// row) and "is it a `TimeShift`" (the shifts directly below a window). +#![warn( + clippy::wildcard_enum_match_arm, + clippy::match_wildcard_for_single_variants +)] use serde::{Deserialize, Serialize}; @@ -389,50 +396,9 @@ pub fn windows_computed_from( if !seen.insert(id) { continue; } - let length_ms = match plan.operator_of(id) { - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => { - u64::try_from(range.as_millis()).ok()? - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => *tumbling_window_width_ms, - Operator::NonASAP( - NonASAPOp::Scan { .. } - | NonASAPOp::Values { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Project { .. } - | NonASAPOp::Aggregate { .. } - | NonASAPOp::Join { .. } - | NonASAPOp::SetOp { .. } - | NonASAPOp::Concat { .. } - | NonASAPOp::Dedup { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::Limit { .. } - | NonASAPOp::BinaryOp { .. } - | NonASAPOp::SQLWindowFunc { .. } - | NonASAPOp::TimeShift { .. } - | NonASAPOp::PromqlVectorFromScalar(..) - | NonASAPOp::PromqlRelabel { .. } - | NonASAPOp::PromqlInfoEnrich { .. } - | NonASAPOp::PromqlSeriesSample { .. } - | NonASAPOp::PromqlSubquery { .. }, - ) - | Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - | ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::SummaryMerge { .. } - | ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. }, - ) => { - pending.extend(plan.inputs_of(id)); - continue; - } + let Some(length_ms) = plan.operator_of(id).covered_window_ms() else { + pending.extend(plan.inputs_of(id)); + continue; }; windows.push(TimeWindow { ends_before_evaluation_ms: shift_back_ms_below(plan, id)?, @@ -469,211 +435,64 @@ mod tests { assert_eq!(shifted.bounds_ms(1_000_000), (640_000, 940_000)); } - /// Every operator's input time access, written out by hand: a new - /// operator needs a row here, as it needs an arm in - /// [`InputTimeAccess::of`]. E.g. `TimeShift(1h)` at query time reads - /// what its readers read, an hour earlier; `Aggregate` reads the rows at - /// t at either timing. `PromqlSubquery` keeps its current access, the - /// rows at t, although it evaluates its input at other times. + /// The operators whose input time access is a decision, not the rows + /// at t: `TimeShift` by timing and lifetime, `@ end`, `TimeRange` and + /// the chain, plus `PromqlSubquery`, which keeps reading the rows at t + /// although it evaluates its input at other times. Pins the operators + /// that exist today; a new operator fails to build in + /// [`InputTimeAccess::of`], and its decision gets a row here. #[test] - fn every_operator_has_its_input_time_access() { - use crate::ir::operator::maintained_population::{ - CurrentSeriesInput, MaintainedPopulation, PopulationInput, PopulationStatistic, - }; - use crate::ir::operator::{ - AggIntent, AtModifier, BinaryOpKind, GroupKeys, JoinKind, Reduction, - RelationalSetOpKind, SampleKind, Source, TimeShift, WindowFuncKind, - }; - use crate::ir::scalar::{ArithmeticOpKind, ColumnRef}; - use crate::ir::schema::{FieldDataType, Schema}; - use crate::ir::{BinaryOperator, Predicate, ScalarExpr, TimeRangeKind}; + fn operators_with_a_time_access_decision() { + use crate::ir::operator::{AggIntent, AtModifier, Reduction, TimeShift}; + use crate::ir::scalar::ColumnRef; + use crate::ir::schema::FieldDataType; + use crate::ir::TimeRangeKind; use crate::stages::StorageMedium; - use crate::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + use crate::summary_type::{SketchAlgorithm, SummaryUpdate}; use std::time::Duration; use ExecutionTiming::{IngestionTime, QueryTime}; use InputRows::*; - let column = || ScalarExpr::::Column(0); - let predicate = || Predicate(column()); let non_asap = |op: NonASAPOp| Operator::NonASAP(op); - let asap = |op: ASAPOp| Operator::ASAP(op); - let kll = || FieldDataType::Sketch(SketchAlgorithm::Kll); - let sample_value = || SummaryUpdate::column(ColumnRef::SampleValue); - let one_hour = |at| TimeShift { - offset_ms: 3_600_000, - at, - }; let shift = |at| { non_asap(NonASAPOp::TimeShift { - shift: one_hour(at), + shift: TimeShift { + offset_ms: 3_600_000, + at, + }, child: 0, }) }; let chain = || { - asap(ASAPOp::SummaryTumblingWindowChain { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: 0, - summary_type: kll(), - input: sample_value(), + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::PerEntity, filter: None, tumbling_window_width_ms: 60_000, }) }; - // Every operator other than `TimeShift`, `TimeRange` and the chain: - // each reads the rows at t, at either timing. - let rows_at_t: Vec> = vec![ - non_asap(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::default(), - }), - non_asap(NonASAPOp::Values { - rows: vec![], - schema: Schema::default(), - }), - non_asap(NonASAPOp::Filter { - pred: predicate(), - child: 0, - }), - non_asap(NonASAPOp::Project { - cols: vec![], - child: 0, - }), - non_asap(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: 0, - }), - non_asap(NonASAPOp::Join { - kind: JoinKind::Inner, - pred: predicate(), - left: 0, - right: 1, - }), - non_asap(NonASAPOp::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left: 0, - right: 1, - }), - non_asap(NonASAPOp::Concat { - children: vec![0, 1], - }), - non_asap(NonASAPOp::Dedup { - cols: vec![0], - child: 0, - }), - non_asap(NonASAPOp::Sort { - keys: vec![], - partition_by: GroupKeys::default(), - child: 0, - }), - non_asap(NonASAPOp::Limit { - n: Some(5), - offset: 0, - partition_by: GroupKeys::default(), - child: 0, - }), - non_asap(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - }, - return_bool: false, - lhs: 0, - rhs: 1, - }), - non_asap(NonASAPOp::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::default(), - order_by: vec![], - frame: None, - output_name: "row".into(), - child: 0, - }), - non_asap(NonASAPOp::PromqlVectorFromScalar(column())), - non_asap(NonASAPOp::PromqlRelabel { - dst: "job".into(), - value: column(), - child: 0, - }), - non_asap(NonASAPOp::PromqlInfoEnrich { - selector: vec![], - child: 0, - }), - non_asap(NonASAPOp::PromqlSeriesSample { - by: GroupKeys::default(), - kind: SampleKind::LimitK(5), - child: 0, - }), - non_asap(NonASAPOp::PromqlSubquery { - range: Duration::from_secs(300), - resolution: None, - child: 0, - }), - asap(ASAPOp::SummaryAgg { - child: 0, - summary_type: kll(), - input: sample_value(), - reduction: Reduction::PerEntity, - filter: None, - }), - asap(ASAPOp::SummaryEstimate { - summary_input: 0, - query: SketchStatistic::Cardinality, - }), - asap(ASAPOp::FinalizeExactAccumulator { child: 0 }), - asap(ASAPOp::MaintainPopulation { - child: 0, - population: MaintainedPopulation { - input: PopulationInput::CurrentSeries(CurrentSeriesInput { - metric: "m".into(), - matchers: vec![], - grouping: vec![], - without: false, - lookback_ms: 300_000, - }), - max_k: 5, - quantiles: false, - }, - }), - asap(ASAPOp::EvaluatePopulation { - child: 0, - evaluation: PopulationStatistic::Sum, - }), - asap(ASAPOp::SummaryMerge { - children: vec![0, 1], - group_by: None, - }), - asap(ASAPOp::SummarySubtract { left: 0, right: 1 }), - asap(ASAPOp::SummaryDelete { - summary_input: 0, - key: 0, - }), - asap(ASAPOp::SummaryJoin { - outer: 0, - inner: 1, - key: 0, - summary_type: kll(), - }), - asap(ASAPOp::Extension { - child: 0, - name: "extension".into(), - }), - ]; + let subquery = non_asap(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(300), + resolution: None, + child: 0, + }); + let aggregate = non_asap(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![String::new()], + filters: vec![], + having: None, + child: 0, + }); let stored = Lifetime::AcrossEvaluations(StorageMedium::Memory); let once = Lifetime::OneEvaluation(StorageMedium::Memory); let access = |reads, shifts_back_ms| InputTimeAccess { reads, shifts_back_ms, }; - let mut table: Vec<(Operator, ExecutionTiming, Lifetime, InputTimeAccess)> = vec![ + let table: Vec<(Operator, ExecutionTiming, Lifetime, InputTimeAccess)> = vec![ ( shift(None), QueryTime, @@ -715,12 +534,9 @@ mod tests { stored, access(LastWindow { length_ms: 60_000 }, 0), ), + (subquery, QueryTime, once, access(CurrentTimeOnly, 0)), + (aggregate, IngestionTime, stored, access(CurrentTimeOnly, 0)), ]; - for operator in rows_at_t { - for timing in [QueryTime, IngestionTime] { - table.push((operator.clone(), timing, stored, access(CurrentTimeOnly, 0))); - } - } for (operator, timing, lifetime, expected) in table { assert_eq!( InputTimeAccess::of(&operator, timing, lifetime), From 7b7d2e1ada3364cbe2c876403b81414f886795f9 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:42:45 +0000 Subject: [PATCH 3/4] refactor: round-2 review fixes for no-silent-defaults - check_raw_data_windows asks covered_window_ms instead of listing every operator; the cost usage `sources` rule takes the same `if let Scan` form as Operator::scanned_sources. - CostModel::unit_costs -> per_node_unit_costs: only cost model 1 prices per node. - InputTimeAccess::of reads TimeRange and chain window lengths from covered_window_ms, so the saturating conversion is written once; the covered_window_ms doc states the saturation change. - keeps_input_time -> keeps_input_window, with a TimeShift example. - Lint comments name every identity check kept (is_asap, lifetime). - Bytes test: note that the executor's unseeded KLL varies by ~0.2%. - dag-viewer fixtures: one output name per measure. --- crates/cost/src/model.rs | 10 ++-- crates/cost/src/node/state_bytes/tests.rs | 4 ++ crates/cost/src/node/usage/mod.rs | 55 +++++-------------- crates/cost/src/plan/cost.rs | 6 +- crates/types/src/ir/operator/node.rs | 25 ++++++--- .../src/plan/materialized_view/raw_data.rs | 51 ++--------------- crates/types/src/plan/time_window.rs | 26 +++++---- .../test-fixtures/memo-enumeration-small.json | 2 +- .../dag-viewer/test-fixtures/memo-small.json | 2 +- 9 files changed, 63 insertions(+), 118 deletions(-) diff --git a/crates/cost/src/model.rs b/crates/cost/src/model.rs index a85f7874b..355867a13 100644 --- a/crates/cost/src/model.rs +++ b/crates/cost/src/model.rs @@ -156,11 +156,11 @@ pub enum CostModel { } impl CostModel { - /// The per-unit costs, which only cost model 1 prices each node's - /// usage with: `UsageOverTime(u)` gives `Some(&u)`; cost model 2 prices - /// the plan's peaks, not each node, and cost model 3 has no prices, so - /// both give `None`. - pub fn unit_costs(&self) -> Option<&ResourceUnitCost> { + /// The per-unit costs each node's usage is priced with. Only cost + /// model 1 prices per node: `UsageOverTime(u)` gives `Some(&u)`. Cost + /// model 2 prices the plan's peaks (in `plan_cost`), not each node, and + /// cost model 3 has no prices, so both give `None`. + pub fn per_node_unit_costs(&self) -> Option<&ResourceUnitCost> { match self { CostModel::UsageOverTime(unit_cost) => Some(unit_cost), CostModel::ProvisionedCapacity(_) | CostModel::ClusterCapacity(_) => None, diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index fae2379e3..add5ba6fe 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -498,6 +498,10 @@ fn priced_bytes_agree_with_the_executor() { }; // (family, its parameters, the rows it is fed, its known gap). let families = [ + // The rows are seeded, but the executor builds KLL unseeded + // (`Kll::with_k`, no seed through its factory), so its compaction + // coin differs per run: the charge varies by about 0.2% (e.g. + // 10,376 or 10,392 bytes), well inside `BYTES_AGREEMENT_FACTOR`. ( sketch(A::Kll), Some(SketchParams::Kll { k: 269 }), diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 66e2d1848..ea0b125b9 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -9,7 +9,9 @@ // "no" by definition: "is it a `Scan`" (a scan's retained raw bytes), and // `raw_data_samples_non_negative`'s "is its producer a `Scan` or a // `TimeRange`" (raw samples; any other producer's are not, the safe -// answer). +// answer). The lifetime check "is it stored across evaluations" +// (`matches!(node.lifetime, Lifetime::AcrossEvaluations(_))`) is kept the +// same way. #![warn( clippy::wildcard_enum_match_arm, clippy::match_wildcard_for_single_variants @@ -310,7 +312,7 @@ pub fn node_resource_usage_and_cost<'a>( let formula_scale = settings.frozen_calibration_factor.unwrap_or(1.0); let table = &settings.measured_operation_costs; let speed = settings.resource_speed; - let unit_costs = settings.cost_model.unit_costs(); + let unit_costs = settings.cost_model.per_node_unit_costs(); let sketch_size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; let mut fit = CalibrationFit::default(); @@ -353,46 +355,15 @@ pub fn node_resource_usage_and_cost<'a>( .map_or(1, |producer| producer.tumbling_windows), _ => 1, }; - let sources: BTreeSet<&str> = match &node.payload { - Payload::NonASAP(NonASAPOp::Scan { source, .. }) => [source_name(source)].into(), - Payload::NonASAP( - NonASAPOp::Values { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Project { .. } - | NonASAPOp::Aggregate { .. } - | NonASAPOp::Join { .. } - | NonASAPOp::SetOp { .. } - | NonASAPOp::Concat { .. } - | NonASAPOp::Dedup { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::Limit { .. } - | NonASAPOp::BinaryOp { .. } - | NonASAPOp::SQLWindowFunc { .. } - | NonASAPOp::TimeRange { .. } - | NonASAPOp::TimeShift { .. } - | NonASAPOp::PromqlVectorFromScalar(..) - | NonASAPOp::PromqlRelabel { .. } - | NonASAPOp::PromqlInfoEnrich { .. } - | NonASAPOp::PromqlSeriesSample { .. } - | NonASAPOp::PromqlSubquery { .. }, - ) - | Payload::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - | ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::SummaryMerge { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. }, - ) => producers - .iter() - .flat_map(|p| p.sources.iter().copied()) - .collect(), - }; + let sources: BTreeSet<&str> = + if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { + [source_name(source)].into() + } else { + producers + .iter() + .flat_map(|p| p.sources.iter().copied()) + .collect() + }; // The series its sources declare, when they all do. let series = declared_series(&sources, data); // The rows per ms its sources receive, which its scans and time diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 34e563225..41253cf88 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -97,7 +97,7 @@ pub fn plan_cost( let mut fit = CalibrationFit::default(); let speed = settings.resource_speed; // Cost model 1 costs each node as it is measured. - let unit_costs = settings.cost_model.unit_costs(); + let unit_costs = settings.cost_model.per_node_unit_costs(); // Each node's `(multiplier, per_second)` ([`computations`]), then its // usage and cost ([`node_resource_usage_and_cost`]), children first. let multipliers = computations(dag, entries, settings.non_repeating_query_horizon_seconds) @@ -400,7 +400,7 @@ pub fn request_overhead_cost_per_second( entries: &[QueryWorkloadEntry], settings: &CostInputs, ) -> Result { - let Some(unit_cost) = settings.cost_model.unit_costs() else { + let Some(unit_cost) = settings.cost_model.per_node_unit_costs() else { return Ok(0.0); }; if settings.measured_operation_costs.ns_per_evaluation <= 0.0 { @@ -422,7 +422,7 @@ pub fn request_overhead_cost_per_second( pub fn raw_data_disk_cost_per_second(bytes: u64, settings: &CostInputs) -> Option { settings .cost_model - .unit_costs() + .per_node_unit_costs() .map(|unit_cost| unit_cost.cost_per_disk_byte_second * bytes as f64) } diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index a0e0da6bb..d17ef7f8c 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -10,7 +10,8 @@ // answer instead of failing the build (under `-D warnings`). Clippy does // not check `if let` and `matches!`, so they stay only for a check of one // variant's identity, where every other operator's answer is "no" by -// definition: "is it a `Scan`" (only a scan reads a source itself). +// definition: "is it a `Scan`" (`scanned_sources`: only a scan reads a +// source itself) and "is it an ASAP operator" (`is_asap`). #![warn( clippy::wildcard_enum_match_arm, clippy::match_wildcard_for_single_variants @@ -125,6 +126,13 @@ impl Operator { /// `Some(60_000)`; `Filter`: `None`. A `PromqlSubquery` evaluates its /// input over a range, but each of its outputs is one evaluation, so it /// covers none here. + /// + /// A `TimeRange` longer than `u64::MAX` ms (about 584 million years) + /// saturates to `Some(u64::MAX)`, so `Some` always means "covers a + /// window"; a caller that once gave `None` for such a range + /// (`windows_computed_from`, and so `RefreshWindow::computed_from`) now + /// carries on with `u64::MAX`. PromQL planning never reaches it: costing such a range + /// overflows first. pub fn covered_window_ms(&self) -> Option { match self { // Saturates, so `Some` means exactly "a window", whatever its length. @@ -217,13 +225,12 @@ impl Operator { } } - /// Whether each output row keeps its input row's time window, so the - /// window its input covers is also its own: `Filter` and `Project` - /// keep rows and times; `TimeShift` moves every row by one offset but - /// keeps the window's length. E.g. `Filter <- TimeRange(5m)` covers - /// 5 minutes, and `Project <- chain of 1-minute windows` holds - /// 1-minute windows; `Aggregate <- TimeRange(5m)` combines rows, so - /// `false`. + /// Whether the window its input covers is also its own, with the same + /// length: `Filter` and `Project` keep each row and its time; + /// `TimeShift` moves every row's time by one offset, so the window + /// moves but keeps its length. E.g. `Filter <- TimeRange(5m)` and + /// `TimeShift(1h) <- TimeRange(5m)` both cover 5 minutes; + /// `Aggregate <- TimeRange(5m)` combines rows, so `false`. pub fn keeps_input_window(&self) -> bool { match self { Operator::NonASAP( @@ -634,7 +641,7 @@ mod tests { /// `Filter`, `Project` and `TimeShift` keep their input's time window; /// an operator that combines rows or covers its own window does not. #[test] - fn filters_projections_and_shifts_keep_input_time() { + fn filters_projections_and_shifts_keep_input_window() { let table = [ (filter(), true), (project(), true), diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index 676e087ac..b17f8ac1e 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -5,7 +5,6 @@ use super::views::ViewIndex; use super::{InstallableDagError, RawDataRetention}; -use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; @@ -117,56 +116,16 @@ impl PhysicalASAPDAGNode { } impl ViewIndex<'_> { - /// See [`PhysicalASAPDAG::check_raw_data_windows`]. Every operator is - /// listed: a new one fails to build here until it is decided whether - /// it holds a window over raw data to check. - #[warn(clippy::wildcard_enum_match_arm)] + /// See [`PhysicalASAPDAG::check_raw_data_windows`]. Only an operator + /// that covers a window ([`Operator::covered_window_ms`]: a `TimeRange` + /// or a tumbling-window chain) holds a window over raw data to check. pub(super) fn check_raw_data_windows(&self) -> Result<(), InstallableDagError> { for node in &self.dag.nodes { if !self.reading_raw_data.contains(&node.id) { continue; } - match &node.payload { - Operator::NonASAP(NonASAPOp::TimeRange { .. }) => { - self.check_raw_window_at(node.id)?; - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - self.check_raw_window_below(node.id)?; - } - // No window of its own over the raw data. - Operator::NonASAP( - NonASAPOp::Scan { .. } - | NonASAPOp::Values { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Project { .. } - | NonASAPOp::Aggregate { .. } - | NonASAPOp::Join { .. } - | NonASAPOp::SetOp { .. } - | NonASAPOp::Concat { .. } - | NonASAPOp::Dedup { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::Limit { .. } - | NonASAPOp::BinaryOp { .. } - | NonASAPOp::SQLWindowFunc { .. } - | NonASAPOp::TimeShift { .. } - | NonASAPOp::PromqlVectorFromScalar(..) - | NonASAPOp::PromqlRelabel { .. } - | NonASAPOp::PromqlInfoEnrich { .. } - | NonASAPOp::PromqlSeriesSample { .. } - | NonASAPOp::PromqlSubquery { .. }, - ) - | Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - | ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::SummaryMerge { .. } - | ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. }, - ) => {} + if node.payload.covered_window_ms().is_some() { + self.check_raw_window_below(node.id)?; } } Ok(()) diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index a2676b021..98dc1f8fd 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -29,7 +29,9 @@ // `matches!` and `let ... else`, so they stay only for a check of one // variant's identity, where every other operator's answer is "no" by // definition: "is it a `Scan`" (only a raw scan's read at t needs every -// row) and "is it a `TimeShift`" (the shifts directly below a window). +// row), "is it a `TimeShift`" (the shifts directly below a window), and +// the lifetime check "is it stored across evaluations" in +// `InputTimeAccess::of` (`matches!(lifetime, Lifetime::AcrossEvaluations(_))`). #![warn( clippy::wildcard_enum_match_arm, clippy::match_wildcard_for_single_variants @@ -140,6 +142,15 @@ impl InputTimeAccess { reads, shifts_back_ms: 0, }; + // The window a `TimeRange` or a chain covers, read in one place + // ([`Operator::covered_window_ms`]); only those two arms call this. + let last_window = || { + reads(InputRows::LastWindow { + length_ms: operator + .covered_window_ms() + .expect("a TimeRange or a tumbling-window chain covers a window"), + }) + }; // Every operator is listed, so a new one fails to build here until // its input time access is decided. match operator { @@ -155,16 +166,9 @@ impl InputTimeAccess { }, Some(_) => reads(InputRows::AllRows), }, - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => reads(InputRows::LastWindow { - length_ms: u64::try_from(range.as_millis()).unwrap_or(u64::MAX), - }), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => match timing { - ExecutionTiming::IngestionTime => reads(InputRows::LastWindow { - length_ms: *tumbling_window_width_ms, - }), + Operator::NonASAP(NonASAPOp::TimeRange { .. }) => last_window(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => match timing { + ExecutionTiming::IngestionTime => last_window(), ExecutionTiming::QueryTime => reads(InputRows::WhatItsReadersRead), }, // A subquery evaluates its input at other times than its own, diff --git a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json index a5bcc3476..d6cfcf314 100644 --- a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json +++ b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} diff --git a/tools/dag-viewer/test-fixtures/memo-small.json b/tools/dag-viewer/test-fixtures/memo-small.json index 2b8af0eb1..8f87de68c 100644 --- a/tools/dag-viewer/test-fixtures/memo-small.json +++ b/tools/dag-viewer/test-fixtures/memo-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} From 4185aff5b7d248dd9160e9cf5922446cc8fd6d21 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:00:44 +0000 Subject: [PATCH 4/4] refactor: rebase onto main: no wildcard arm in main's new matches under the module lint Main added a wildcard arm in shift_back_ms_below (time_window.rs) and in the tumbling-windows match (node::usage), both modules this PR lints with clippy::wildcard_enum_match_arm. The first is the 'is it a TimeShift' identity check the module comment already allows, now an if let; the second names both Operator halves, since keeps_input_window answers per operator. --- crates/cost/src/node/usage/mod.rs | 4 +++- crates/types/src/plan/time_window.rs | 13 ++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index ea0b125b9..5288d954f 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -353,7 +353,9 @@ pub fn node_resource_usage_and_cost<'a>( payload if payload.keeps_input_window() => producers .first() .map_or(1, |producer| producer.tumbling_windows), - _ => 1, + // Any other operator spans one window: `keeps_input_window` + // answers per operator, so listing them here would decide nothing. + Payload::NonASAP(_) | Payload::ASAP(_) => 1, }; let sources: BTreeSet<&str> = if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index 98dc1f8fd..f6c960fc0 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -361,14 +361,13 @@ pub fn shift_back_ms_below(plan: &P, node: P::Node) -> Optio // shift. let mut path_ends = inputs.is_empty(); for input in inputs { - match plan.operator_of(input) { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - if shift.at.is_some() { - return None; - } - pending.push((input, walked_ms.checked_add(shift.offset_ms)?)); + if let Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) = plan.operator_of(input) { + if shift.at.is_some() { + return None; } - _ => path_ends = true, + pending.push((input, walked_ms.checked_add(shift.offset_ms)?)); + } else { + path_ends = true; } } if path_ends {