diff --git a/crates/cost/Cargo.toml b/crates/cost/Cargo.toml index 5bbdc691c..e0298ba38 100644 --- a/crates/cost/Cargo.toml +++ b/crates/cost/Cargo.toml @@ -16,5 +16,6 @@ serde_json = "1" [dev-dependencies] # The executor's charge for each summary state, checked against the bytes -# the state byte models give for it plus the accumulator's own fields (#947). +# the state byte models give for it plus the accumulator's own fields, and +# against priced bytes on seeded workload data (#947). asap-executor = { path = "../executor" } diff --git a/crates/cost/src/model.rs b/crates/cost/src/model.rs index caba6b63b..355867a13 100644 --- a/crates/cost/src/model.rs +++ b/crates/cost/src/model.rs @@ -155,6 +155,19 @@ pub enum CostModel { ClusterCapacity(ClusterLimits), } +impl CostModel { + /// The per-unit costs each node's usage is priced with. Only cost + /// model 1 prices per node: `UsageOverTime(u)` gives `Some(&u)`. Cost + /// model 2 prices the plan's peaks (in `plan_cost`), not each node, and + /// cost model 3 has no prices, so both give `None`. + pub fn per_node_unit_costs(&self) -> Option<&ResourceUnitCost> { + match self { + CostModel::UsageOverTime(unit_cost) => Some(unit_cost), + CostModel::ProvisionedCapacity(_) | CostModel::ClusterCapacity(_) => None, + } + } +} + /// A fixed cluster's capacity (cost model 3) and how to choose within it. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index 3e30a6bb2..add5ba6fe 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -24,6 +24,68 @@ fn exact_state_is_priced_at_the_executors_state_size() { assert_eq!(asap_types::summary_type::EXACT_ACCUMULATOR_STATE_BYTES, 56); } +/// The rows a family is fed. +enum Rows { + /// One sample value a row. + Values, + /// One label value a row, as Pass 1 feeds UnivMon a column. + Labels, + /// A label set and a weight a row. + Keyed, +} + +impl Rows { + /// How many rows [`priced_bytes_agree_with_the_executor`] feeds a + /// family of these rows, priced for the same count. 10,000 values: a + /// KLL state's charge reaches its priced size only past about 3,000 + /// rows. 1,000 label rows: the label families' grids have a fixed + /// size, so their ratio is the same at any count, and 1,000 rows over + /// 1,000 keys still churn a heap of 10; at 10,000 rows Count Sketch + /// with a heap alone took about 22 s in a debug build. + fn fed(&self) -> u32 { + match self { + Rows::Values => 10_000, + Rows::Labels | Rows::Keyed => 1_000, + } + } + + /// The update a family fed these rows reads: the sample value; an + /// `item` label counted once a row; or an `item` label set weighted by + /// the sample value. + fn update(&self) -> asap_types::summary_type::SummaryUpdate { + use asap_types::ir::scalar::expr_ir::ColumnRef; + use asap_types::summary_type::{SummaryInputExpr, SummaryUpdate}; + let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); + match self { + Rows::Values => SummaryUpdate::column(ColumnRef::SampleValue), + Rows::Labels => SummaryUpdate { + item, + weight: SummaryInputExpr::Constant(1.0), + }, + Rows::Keyed => SummaryUpdate { + item, + ..SummaryUpdate::column(ColumnRef::SampleValue) + }, + } + } +} + +/// A 16-byte label value, `i` zero-padded: its key has +/// [`ASSUMED_VALUE_KEY_BYTES`]. +fn label(i: u32) -> String { + format!("{i:016}") +} + +/// Three 16-byte labels, the first [`label`]`(i)`: a tuple key of +/// [`ASSUMED_HEAP_KEY_BYTES`]. +fn labels(i: u32) -> [String; 3] { + [ + label(i), + "service-frontend".to_string(), + "region-us-east-1".to_string(), + ] +} + /// The bytes `asap_sketchlib`'s DDSketch holds after adding `values`. fn library_ddsketch_bytes(alpha: f64, values: &[f64]) -> u64 { let mut sketch = asap_sketchlib::sketches::DdSketch::new(alpha); @@ -143,7 +205,8 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { /// family. It equals the priced bytes where `asap_sketchlib`'s estimate /// is exact (Count-Min, Count Sketch, HLL, KMV at `n` distinct items) and /// for KLL up to `k` rows. For DDSketch, the heap sketches and UnivMon -/// the price bounds it, by at most 1.5 times once the state is full; +/// the price bounds it, by at most [`BYTES_AGREEMENT_FACTOR`] times once +/// the state is full; /// UnivMon's price also bounds the bytes its sketch holds. /// KLL past `k` rows is checked only loosely: the planner's formula /// leaves out the compaction scratch space, a known gap until a @@ -159,8 +222,6 @@ fn state_bytes_bound_the_executors_charge() { use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; use asap_executor::summary_kernels::HllSketchAccumulator; use asap_executor::values::Value; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::summary_type::{SummaryInputExpr, SummaryUpdate}; use std::mem::{align_of, size_of}; use SketchAlgorithm as A; /// How the priced bytes relate to the sketch's. @@ -171,18 +232,10 @@ fn state_bytes_bound_the_executors_charge() { Kll { k: u32, }, - /// At least, and at most 1.5 times once the state is full. + /// At least, and at most [`BYTES_AGREEMENT_FACTOR`] times once the + /// state is full. Bound, } - /// The rows a family is fed. - enum Rows { - /// Signed values over the assumed range. - Values, - /// One label value a row, as Pass 1 feeds UnivMon a column. - Labels, - /// A label set and a weight a row. - Keyed, - } // (family, parameters, rows fed, how it is priced, the // accumulator's own fields, rows at which the state is full: four // times the heap for the heap sketches, so each heap churns). @@ -291,30 +344,10 @@ fn state_bytes_bound_the_executors_charge() { 4_096, ), ]; - let value = SummaryUpdate::column(ColumnRef::SampleValue); - let item = Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))); - let keyed = SummaryUpdate { - item: item.clone(), - ..value.clone() - }; - let labeled = SummaryUpdate { - item, - weight: SummaryInputExpr::Constant(1.0), - }; - // A 16-byte label value: its key has the assumed size. - let label = |i: u32| format!("{i:016}"); assert_eq!( Value::Utf8(label(0).into()).key().unwrap().len(), ASSUMED_VALUE_KEY_BYTES ); - // Three 16-byte labels: a tuple key of the assumed size. - let labels = |i: u32| { - [ - label(i), - "service-frontend".to_string(), - "region-us-east-1".to_string(), - ] - }; let key = asap_sketchlib::TupleKey::new( labels(0) .into_iter() @@ -325,12 +358,8 @@ fn state_bytes_bound_the_executors_charge() { for (algorithm, params, rows, priced_as, fields, full) in &families { for n in [1, 100, *full] { let family = FieldDataType::Sketch(*algorithm); - let input = match rows { - Rows::Values => &value, - Rows::Labels => &labeled, - Rows::Keyed => &keyed, - }; - let mut updater = create_planner_accumulator(&family, Some(params), input).unwrap(); + let mut updater = + create_planner_accumulator(&family, Some(params), &rows.update()).unwrap(); for i in 0..n { match rows { // Distinct magnitudes over the assumed range, in a @@ -372,7 +401,7 @@ fn state_bytes_bound_the_executors_charge() { assert!(charged <= priced, "{context}, {charged} charged"); if n == *full { assert!( - priced as f64 <= 1.5 * charged as f64, + priced as f64 <= BYTES_AGREEMENT_FACTOR * charged as f64, "{context}, {charged} charged" ); } @@ -383,7 +412,8 @@ fn state_bytes_bound_the_executors_charge() { // The estimate is a heap whose digest index has grown once // more under churn, which the test's labels do not reach: a // full heap holds about two thirds of it (1.52 times less - // here), so this bound is 1.6 times rather than 1.5. + // here), so this bound is 1.6 times rather than + // `BYTES_AGREEMENT_FACTOR`. if let Some(univmon) = state.as_any().downcast_ref::() { let held = fields + univmon.sketch().memory_bytes(); assert!(held <= charged, "{context}, {held} held"); @@ -394,3 +424,254 @@ fn state_bytes_bound_the_executors_charge() { } } } + +/// How far plan selection's bytes for a state may be from what the +/// executor charges for it: priced ÷ charged in `[1 / 1.5, 1.5]` on +/// seeded data ([`priced_bytes_agree_with_the_executor`]), and priced at +/// most 1.5 times the charge once a worst-case state is full +/// ([`state_bytes_bound_the_executors_charge`]). +const BYTES_AGREEMENT_FACTOR: f64 = 1.5; + +/// A family's priced bytes known to disagree with the executor's beyond +/// [`BYTES_AGREEMENT_FACTOR`], on its row of +/// [`priced_bytes_agree_with_the_executor`]: the issue that fixes it and +/// priced ÷ charged as measured there. The test fails when the family +/// agrees (remove the gap) or its ratio moves more than 5% off this one. +struct KnownBytesGap { + issue: u32, + priced_over_charged: f64, +} + +/// The next value of the seeded stream `state` (splitmix64), uniform in +/// `[0, 1)`, so the rows every family is fed are fixed. +fn next_uniform(state: &mut u64) -> f64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + ((z ^ (z >> 31)) >> 11) as f64 / (1u64 << 53) as f64 +} + +/// Plan selection prices each state at what the executor charges for it +/// on data shaped like a workload's, not only on the worst case +/// [`state_bytes_bound_the_executors_charge`] feeds. Every family the +/// executor builds from a plan, at Pass 1's parameters for a 1% error, is +/// built through the executor and fed seeded rows ([`Rows::fed`]): +/// 10,000 latencies log-uniform over 1 ms to 1 s (declared +/// non-negative), or 1,000 rows of 1,000 distinct label values or label +/// sets. Its charge +/// (`approx_memory_bytes`; for HydraCms its shared grid over its 10 +/// groups) is compared with `summary_shape` for the same rows. Priced ÷ +/// charged lies within [`BYTES_AGREEMENT_FACTOR`], or the family's row +/// carries its [`KnownBytesGap`]. +/// +/// HydraKll is left out: no plan builds it (it has no parameter list, +/// and the executor's factory refuses it). +#[test] +fn priced_bytes_agree_with_the_executor() { + use asap_executor::factory::{create_hydra_cms, create_planner_accumulator}; + use asap_executor::values::Value; + use asap_types::summary_type::{ExactKind, ExactParams, HydraParams}; + use SketchAlgorithm as A; + const DISTINCT_KEYS: u32 = 1_000; + const HYDRA_GROUPS: u32 = 10; + let input_rows = |rows_per_state: u32| StateInputRows { + rows_per_state: Some(u64::from(rows_per_state)), + non_negative: true, + }; + let gap = |issue, priced_over_charged| { + Some(KnownBytesGap { + issue, + priced_over_charged, + }) + }; + let sketch = FieldDataType::Sketch; + // Priced as one 56-byte record of the executor's exact state column; + // the accumulator object built here charges its whole 136-byte struct. + let exact = |kind, params| { + ( + FieldDataType::ExactAggregate(kind, params), + None, + Rows::Values, + gap(1333, 0.412), + ) + }; + // (family, its parameters, the rows it is fed, its known gap). + let families = [ + // The rows are seeded, but the executor builds KLL unseeded + // (`Kll::with_k`, no seed through its factory), so its compaction + // coin differs per run: the charge varies by about 0.2% (e.g. + // 10,376 or 10,392 bytes), well inside `BYTES_AGREEMENT_FACTOR`. + ( + sketch(A::Kll), + Some(SketchParams::Kll { k: 269 }), + Rows::Values, + None, + ), + // Priced for values over nine decades + // (`DDSKETCH_ASSUMED_VALUE_RANGE`); these span three. + ( + sketch(A::DDSketch), + Some(SketchParams::DDSketch { alpha: 0.01 }), + Rows::Values, + gap(1313, 2.982), + ), + ( + sketch(A::Hll), + Some(SketchParams::Hll { precision: 18 }), + Rows::Values, + None, + ), + ( + sketch(A::Kmv), + Some(SketchParams::Kmv { k: 106_263 }), + Rows::Values, + None, + ), + ( + sketch(A::Cms), + Some(SketchParams::Cms { + width: 272, + depth: 5, + }), + Rows::Keyed, + None, + ), + ( + sketch(A::CountSketch), + Some(SketchParams::CountSketch { + width: 30_000, + depth: 47, + }), + Rows::Keyed, + None, + ), + ( + sketch(A::CmsWithHeap), + Some(SketchParams::CmsWithHeap { + width: 272, + depth: 5, + heap_size: 10, + }), + Rows::Keyed, + None, + ), + ( + sketch(A::CountSketchWithHeap), + Some(SketchParams::CountSketchWithHeap { + width: 30_000, + depth: 47, + heap_size: 10, + }), + Rows::Keyed, + None, + ), + ( + sketch(A::UnivMon), + Some(SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1_024, + layers: 16, + }), + Rows::Labels, + None, + ), + exact(ExactKind::Sum, ExactParams::Sum), + exact(ExactKind::Count, ExactParams::Count), + exact(ExactKind::Min, ExactParams::Min), + exact(ExactKind::Max, ExactParams::Max), + exact(ExactKind::Increase, ExactParams::Increase), + exact(ExactKind::Rate, ExactParams::Rate), + ]; + // Every family the executor declares is checked. + for supported in asap_executor::supported_summary_families() { + assert!( + families + .iter() + .any(|(family, ..)| *family == sketch(supported.summary_type)), + "{:?} is built by the executor but not checked here", + supported.summary_type + ); + } + // (family, priced bytes, charged bytes, its known gap). + let mut measured = Vec::new(); + for (family, params, rows, known_gap) in families { + let rows_fed = rows.fed(); + let mut updater = + create_planner_accumulator(&family, params.as_ref(), &rows.update()).unwrap(); + let mut seed = 0x5EED_u64; + for i in 0..rows_fed { + let key = (next_uniform(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; + match rows { + // One sample a second, its latency log-uniform over 1 ms to 1 s. + Rows::Values => { + let latency_ms = 1_000f64.powf(next_uniform(&mut seed)); + updater + .update_single(latency_ms, i64::from(i) * 1_000) + .unwrap(); + } + Rows::Labels => updater + .update_value(&Value::Utf8(label(key).into()), 0) + .unwrap(), + Rows::Keyed => updater.update_keyed(&labels(key), 1.0, 0).unwrap(), + } + } + let charged = updater.into_accumulator().approx_memory_bytes() as u64; + let priced = summary_shape(&family, params.as_ref(), input_rows(rows_fed)) + .unwrap() + .1; + measured.push((family, priced, charged, known_gap)); + } + // HydraCms: one grid for every group, priced as one Count-Min per + // group; the executor keeps one grid for all of them, here 3 × 64 + // cells over 10 groups. + let hydra = SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 64, + }); + let hydra_family = sketch(A::HydraCms); + let mut grid = create_hydra_cms(&hydra_family, Some(&hydra)).unwrap(); + let hydra_rows_fed = Rows::Keyed.fed(); + let mut seed = 0x5EED_u64; + for _ in 0..hydra_rows_fed { + let group = (next_uniform(&mut seed) * f64::from(HYDRA_GROUPS)) as u32; + let item = (next_uniform(&mut seed) * f64::from(DISTINCT_KEYS)) as u32; + grid.count(&label(group), &label(item), 1).unwrap(); + } + measured.push(( + hydra_family, + sketch_shape(&hydra, input_rows(hydra_rows_fed / HYDRA_GROUPS)).1, + (grid.approx_memory_bytes() / HYDRA_GROUPS as usize) as u64, + gap(947, 0.104), + )); + + let mut failures = Vec::new(); + for (family, priced, charged, known_gap) in &measured { + let ratio = *priced as f64 / *charged as f64; + let agrees = (1.0 / BYTES_AGREEMENT_FACTOR..=BYTES_AGREEMENT_FACTOR).contains(&ratio); + let line = format!( + "{}: {priced} priced, {charged} charged, ratio {ratio:.3}", + crate::node::usage::summary_type_name(family) + ); + match known_gap { + Some(KnownBytesGap { issue, .. }) if agrees => { + failures.push(format!("{line}: agrees now; remove its known gap (#{issue})")) + } + Some(KnownBytesGap { + issue, + priced_over_charged, + }) if (ratio / priced_over_charged - 1.0).abs() > 0.05 => failures.push(format!( + "{line}: its known gap (#{issue}) is {priced_over_charged}; update it" + )), + Some(_) => {} + None if !agrees => failures.push(format!( + "{line}: outside {BYTES_AGREEMENT_FACTOR}x; fix it or give its row a known gap with its issue" + )), + None => {} + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index f347c1259..5288d954f 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -1,6 +1,22 @@ //! One node of a physical plan DAG: its resource usage and cost //! ([`node_resource_usage_and_cost`], [`NodeCost`]). +// The rules here answer per operator, summary type or cost model, and +// each lists every variant: a wildcard arm would give a new variant a +// default answer instead of failing the build (under `-D warnings`). +// Clippy does not check `matches!` and `if let`, so they stay only for a +// check of one variant's identity, where every other variant's answer is +// "no" by definition: "is it a `Scan`" (a scan's retained raw bytes), and +// `raw_data_samples_non_negative`'s "is its producer a `Scan` or a +// `TimeRange`" (raw samples; any other producer's are not, the safe +// answer). The lifetime check "is it stored across evaluations" +// (`matches!(node.lifetime, Lifetime::AcrossEvaluations(_))`) is kept the +// same way. +#![warn( + clippy::wildcard_enum_match_arm, + clippy::match_wildcard_for_single_variants +)] + use std::collections::{BTreeMap, BTreeSet}; use crate::node::analytical::{ @@ -14,8 +30,8 @@ use crate::node::state_bytes::{ }; use crate::{aggregate_lookup, lookup, StateFamily}; use crate::{ - CalibrationFit, CostInputs, CostModel, NodeCostSource, OperatorKind, COST_SOURCE, - MEASURED_COST_SOURCE, MIXED_COST_SOURCE, + CalibrationFit, CostInputs, NodeCostSource, OperatorKind, COST_SOURCE, MEASURED_COST_SOURCE, + MIXED_COST_SOURCE, }; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; use asap_types::ir::properties::ExecutionTiming; @@ -93,14 +109,46 @@ fn state_family<'a>( .map(StateFamily::Sketch) .ok_or(AnalyticalCostError::UnsupportedCandidate)?, FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), - _ => StateFamily::None, + FieldDataType::Plain(_) + | FieldDataType::SummarySample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..) => StateFamily::None, }, Payload::ASAP( ASAPOp::SummaryMerge { .. } | ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }, ) => producer_family.unwrap_or(StateFamily::None), - _ => StateFamily::None, + Payload::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => StateFamily::None, }) } @@ -264,10 +312,7 @@ pub fn node_resource_usage_and_cost<'a>( let formula_scale = settings.frozen_calibration_factor.unwrap_or(1.0); let table = &settings.measured_operation_costs; let speed = settings.resource_speed; - let unit_costs = match &settings.cost_model { - CostModel::UsageOverTime(unit_cost) => Some(unit_cost), - _ => None, - }; + let unit_costs = settings.cost_model.per_node_unit_costs(); let sketch_size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; let mut fit = CalibrationFit::default(); @@ -308,15 +353,19 @@ pub fn node_resource_usage_and_cost<'a>( payload if payload.keeps_input_window() => producers .first() .map_or(1, |producer| producer.tumbling_windows), - _ => 1, - }; - let sources: BTreeSet<&str> = match &node.payload { - Payload::NonASAP(NonASAPOp::Scan { source, .. }) => [source_name(source)].into(), - _ => producers - .iter() - .flat_map(|p| p.sources.iter().copied()) - .collect(), + // Any other operator spans one window: `keeps_input_window` + // answers per operator, so listing them here would decide nothing. + Payload::NonASAP(_) | Payload::ASAP(_) => 1, }; + let sources: BTreeSet<&str> = + if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { + [source_name(source)].into() + } else { + producers + .iter() + .flat_map(|p| p.sources.iter().copied()) + .collect() + }; // The series its sources declare, when they all do. let series = declared_series(&sources, data); // The rows per ms its sources receive, which its scans and time @@ -373,7 +422,40 @@ pub fn node_resource_usage_and_cost<'a>( Payload::NonASAP(NonASAPOp::Aggregate { measures, .. }) => { aggregate_lookup(table, node.output_state.timing, measures) } - _ => lookup( + Payload::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => lookup( table, operator, node.output_state.timing, @@ -530,7 +612,7 @@ pub fn node_resource_usage_and_cost<'a>( let group_count = groups(reduction)?; let keys = match reduction { Reduction::Reduce(keys) => keys.keys().len() as u64, - _ => 1, + Reduction::PerEntity => 1, }; let out = edge(group_count)?; let held = input.rows.div_ceil(group_count.max(1)); @@ -696,7 +778,20 @@ pub fn node_resource_usage_and_cost<'a>( ), ) } - _ => { + NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. } => { let estimate = estimate_operator( PhysicalOperator::PassThrough, OperatorStatistics::PassThrough { @@ -827,7 +922,11 @@ pub fn node_resource_usage_and_cost<'a>( Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { let per_group = match query { SketchStatistic::TopK { k } => *k as u64, - _ => 1, + SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy + | SketchStatistic::Quantile { .. } + | SketchStatistic::PointCount { .. } + | SketchStatistic::Cardinality => 1, }; let out = edge( input @@ -846,7 +945,14 @@ pub fn node_resource_usage_and_cost<'a>( Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), format!("finalize {} accumulators", input.rows), ), - _ => ( + Payload::ASAP( + ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => ( edge(input.rows)?, Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), format!("{} rows", input.rows), @@ -862,7 +968,39 @@ pub fn node_resource_usage_and_cost<'a>( let units = match &node.payload { Payload::NonASAP(NonASAPOp::Scan { .. }) => out.rows, Payload::ASAP(ASAPOp::SummaryMerge { .. }) => merged_states(producers)?, - _ => input.rows, + Payload::NonASAP( + NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => input.rows, }; let mut detail = detail; // A node with an entry takes its measured time, scanning included; @@ -1102,6 +1240,9 @@ fn merged_states(producers: &[ProducerOutput<'_>]) -> Result String { match summary_type { FieldDataType::Sketch(algorithm) => format!("{algorithm:?}"), diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 47922c71c..41253cf88 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -97,10 +97,7 @@ pub fn plan_cost( let mut fit = CalibrationFit::default(); let speed = settings.resource_speed; // Cost model 1 costs each node as it is measured. - let unit_costs = match &settings.cost_model { - CostModel::UsageOverTime(unit_cost) => Some(unit_cost), - _ => None, - }; + let unit_costs = settings.cost_model.per_node_unit_costs(); // Each node's `(multiplier, per_second)` ([`computations`]), then its // usage and cost ([`node_resource_usage_and_cost`]), children first. let multipliers = computations(dag, entries, settings.non_repeating_query_horizon_seconds) @@ -403,7 +400,7 @@ pub fn request_overhead_cost_per_second( entries: &[QueryWorkloadEntry], settings: &CostInputs, ) -> Result { - let CostModel::UsageOverTime(unit_cost) = &settings.cost_model else { + let Some(unit_cost) = settings.cost_model.per_node_unit_costs() else { return Ok(0.0); }; if settings.measured_operation_costs.ns_per_evaluation <= 0.0 { @@ -423,12 +420,10 @@ pub fn request_overhead_cost_per_second( /// `None` under cost models 2 and 3. A plan keeps each source once, for its /// scan that keeps the most ([`plan_cost`]). pub fn raw_data_disk_cost_per_second(bytes: u64, settings: &CostInputs) -> Option { - match &settings.cost_model { - CostModel::UsageOverTime(unit_cost) => { - Some(unit_cost.cost_per_disk_byte_second * bytes as f64) - } - _ => None, - } + settings + .cost_model + .per_node_unit_costs() + .map(|unit_cost| unit_cost.cost_per_disk_byte_second * bytes as f64) } /// The window a query-time `Scan` `node` of `dag` reads per execution: the diff --git a/crates/executor/src/summary_kernels/traits.rs b/crates/executor/src/summary_kernels/traits.rs index 372adeac1..b7aa08e57 100644 --- a/crates/executor/src/summary_kernels/traits.rs +++ b/crates/executor/src/summary_kernels/traits.rs @@ -35,10 +35,9 @@ pub trait AggregateCore: Send + Sync { /// Approximate in-memory footprint, used for execution memory reservations. /// A state shared with other rows (see [`Self::shared_state`]) is not - /// included. - fn approx_memory_bytes(&self) -> usize { - 4096 - } + /// included. No default: each kernel says what it holds, e.g. a KLL + /// state its sketch's `memory_bytes()`. + fn approx_memory_bytes(&self) -> usize; /// The identity and footprint of a state this row shares with other /// rows (one Hydra grid read by every group of a build). Reservations diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 7fa100936..d17ef7f8c 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -5,6 +5,18 @@ //! each physical candidate holds its own //! ([`crate::stages::PhysicalCandidate::properties`]). +// The questions answered here per operator (e.g. the window it covers) +// list every operator: a wildcard arm would give a new operator a default +// answer instead of failing the build (under `-D warnings`). Clippy does +// not check `if let` and `matches!`, so they stay only for a check of one +// variant's identity, where every other operator's answer is "no" by +// definition: "is it a `Scan`" (`scanned_sources`: only a scan reads a +// source itself) and "is it an ASAP operator" (`is_asap`). +#![warn( + clippy::wildcard_enum_match_arm, + clippy::match_wildcard_for_single_variants +)] + use std::collections::{BTreeSet, HashSet}; use std::rc::Rc; @@ -108,6 +120,65 @@ impl Operator { } } + /// The window of input rows one output covers, in ms: a `TimeRange`'s + /// range or a chain's tumbling window; `None` for every other operator. + /// E.g. `TimeRange(5m)`: `Some(300_000)`; a chain of 1-minute windows: + /// `Some(60_000)`; `Filter`: `None`. A `PromqlSubquery` evaluates its + /// input over a range, but each of its outputs is one evaluation, so it + /// covers none here. + /// + /// A `TimeRange` longer than `u64::MAX` ms (about 584 million years) + /// saturates to `Some(u64::MAX)`, so `Some` always means "covers a + /// window"; a caller that once gave `None` for such a range + /// (`windows_computed_from`, and so `RefreshWindow::computed_from`) now + /// carries on with `u64::MAX`. PromQL planning never reaches it: costing such a range + /// overflows first. + pub fn covered_window_ms(&self) -> Option { + match self { + // Saturates, so `Some` means exactly "a window", whatever its length. + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => { + Some(u64::try_from(range.as_millis()).unwrap_or(u64::MAX)) + } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => None, + } + } + /// The anchor of a `TimeShift` with `@ end()` or `@ start()`, whose /// input runs once per request; `None` for any other operator (`@ ` /// included). @@ -154,13 +225,12 @@ impl Operator { } } - /// Whether each output row keeps its input row's time window, so the - /// window its input covers is also its own: `Filter` and `Project` - /// keep rows and times; `TimeShift` moves every row by one offset but - /// keeps the window's length. E.g. `Filter <- TimeRange(5m)` covers - /// 5 minutes, and `Project <- chain of 1-minute windows` holds - /// 1-minute windows; `Aggregate <- TimeRange(5m)` combines rows, so - /// `false`. + /// Whether the window its input covers is also its own, with the same + /// length: `Filter` and `Project` keep each row and its time; + /// `TimeShift` moves every row's time by one offset, so the window + /// moves but keeps its length. E.g. `Filter <- TimeRange(5m)` and + /// `TimeShift(1h) <- TimeRange(5m)` both cover 5 minutes; + /// `Aggregate <- TimeRange(5m)` combines rows, so `false`. pub fn keeps_input_window(&self) -> bool { match self { Operator::NonASAP( @@ -209,14 +279,13 @@ impl Operator { /// own, else the union over its inputs' /// ([`OperatorInput::scanned_sources`]). pub(crate) fn scanned_sources(&self) -> BTreeSet { - match self { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), - operator => operator - .children() - .into_iter() - .flat_map(OperatorInput::scanned_sources) - .collect(), + if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = self { + return BTreeSet::from([source.clone()]); } + self.children() + .into_iter() + .flat_map(OperatorInput::scanned_sources) + .collect() } pub(crate) fn output_schema(&self) -> Result { @@ -440,3 +509,150 @@ impl OperatorNode { Ok(()) } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::ir::operator::{AggIntent, Reduction, TimeShift}; + use crate::ir::scalar::ColumnRef; + use crate::ir::schema::FieldDataType; + use crate::ir::{Predicate, ScalarExpr, TimeRangeKind}; + use crate::summary_type::{SketchAlgorithm, SummaryUpdate}; + + fn scan() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::default(), + }) + } + + fn filter() -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Column(0)), + child: 0, + }) + } + + fn project() -> Operator { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![], + child: 0, + }) + } + + fn time_shift_one_hour() -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: 0, + }) + } + + fn aggregate() -> Operator { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![String::new()], + filters: vec![], + having: None, + child: 0, + }) + } + + fn time_range_five_minutes() -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: 0, + }) + } + + fn subquery_five_minutes() -> Operator { + Operator::NonASAP(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(300), + resolution: None, + child: 0, + }) + } + + fn summary_agg() -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) + } + + fn chain_one_minute() -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }) + } + + /// Only a chain has a tumbling-window width; a `TimeRange` covers a + /// window but tumbles none. + #[test] + fn only_a_chain_has_a_tumbling_window_width() { + let table = [ + (chain_one_minute(), Some(60_000)), + (time_range_five_minutes(), None), + (summary_agg(), None), + (filter(), None), + ]; + for (operator, expected) in table { + assert_eq!( + operator.chain_tumbling_window_width_ms(), + expected, + "{operator:?}" + ); + } + } + + /// A `TimeRange` covers its range and a chain its tumbling window; a + /// subquery's range is not a covered window, as each of its outputs is + /// one evaluation. + #[test] + fn a_time_range_and_a_chain_cover_a_window() { + let table = [ + (time_range_five_minutes(), Some(300_000)), + (chain_one_minute(), Some(60_000)), + (subquery_five_minutes(), None), + (time_shift_one_hour(), None), + (aggregate(), None), + (scan(), None), + ]; + for (operator, expected) in table { + assert_eq!(operator.covered_window_ms(), expected, "{operator:?}"); + } + } + + /// `Filter`, `Project` and `TimeShift` keep their input's time window; + /// an operator that combines rows or covers its own window does not. + #[test] + fn filters_projections_and_shifts_keep_input_window() { + let table = [ + (filter(), true), + (project(), true), + (time_shift_one_hour(), true), + (aggregate(), false), + (time_range_five_minutes(), false), + (chain_one_minute(), false), + (scan(), false), + ]; + for (operator, expected) in table { + assert_eq!(operator.keeps_input_window(), expected, "{operator:?}"); + } + } +} diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 49a361f31..c3da702dd 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -82,7 +82,8 @@ pub enum NonASAPOp> { measures: Vec, /// Output column names, exactly one entry per measure; an empty /// string means the default name (the intent's synthetic name). - #[serde(default)] + /// Required when deserialized: a missing list would not match the + /// measures. output_names: Vec, #[serde(default)] filters: Vec>>, @@ -1446,6 +1447,34 @@ mod tests { } } + /// An `Aggregate` read without `output_names` is rejected instead of + /// getting an empty list that matches none of its measures; with its + /// names it round-trips. E.g. `{"Aggregate": {"measures": ["Rate"], + /// ...}}` with no `"output_names"` fails with a missing field. + #[test] + fn aggregate_output_names_are_required_when_deserialized() { + let aggregate = rate_over(scan( + vec![col("value", DataType::Float64, false)], + None, + vec![], + )); + let json = serde_json::to_value(&aggregate).unwrap(); + let read_back: NonASAPOp = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(read_back, aggregate); + + let mut without_names = json; + without_names["Aggregate"] + .as_object_mut() + .unwrap() + .remove("output_names") + .unwrap(); + let error = serde_json::from_value::(without_names).unwrap_err(); + assert!( + error.to_string().contains("missing field `output_names`"), + "{error}" + ); + } + #[test] fn project_preserves_unique_keys_that_are_passed_through() { let input = scan( diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index 707899c13..b17f8ac1e 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -5,7 +5,6 @@ use super::views::ViewIndex; use super::{InstallableDagError, RawDataRetention}; -use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; @@ -117,20 +116,16 @@ impl PhysicalASAPDAGNode { } impl ViewIndex<'_> { - /// See [`PhysicalASAPDAG::check_raw_data_windows`]. + /// See [`PhysicalASAPDAG::check_raw_data_windows`]. Only an operator + /// that covers a window ([`Operator::covered_window_ms`]: a `TimeRange` + /// or a tumbling-window chain) holds a window over raw data to check. pub(super) fn check_raw_data_windows(&self) -> Result<(), InstallableDagError> { for node in &self.dag.nodes { if !self.reading_raw_data.contains(&node.id) { continue; } - match &node.payload { - Operator::NonASAP(NonASAPOp::TimeRange { .. }) => { - self.check_raw_window_at(node.id)?; - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - self.check_raw_window_below(node.id)?; - } - _ => {} + if node.payload.covered_window_ms().is_some() { + self.check_raw_window_below(node.id)?; } } Ok(()) diff --git a/crates/types/src/plan/materialized_view/views.rs b/crates/types/src/plan/materialized_view/views.rs index 835abdc39..5348bf15b 100644 --- a/crates/types/src/plan/materialized_view/views.rs +++ b/crates/types/src/plan/materialized_view/views.rs @@ -200,11 +200,12 @@ impl<'a> ViewIndex<'a> { .flatten() .any(|input| match payload(input) { Some(Operator::NonASAP(NonASAPOp::Scan { .. })) => true, - Some( - Operator::NonASAP(NonASAPOp::TimeRange { .. }) - | Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ) => false, - _ => reading_raw_data.contains(input), + // A window (`TimeRange`, chain) between it and the raw + // data: it reads the window's output, not raw rows. + Some(operator) if operator.covered_window_ms().is_some() => false, + // Any other operator passes on whether its input reads + // raw data. + Some(_) | None => reading_raw_data.contains(input), }); if reads { reading_raw_data.insert(*id); diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 7e91ce4b5..afc2ae94b 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -567,6 +567,10 @@ impl PhysicalASAPDAG { /// A node referenced several times from the scalar expressions is one /// `ScalarRef` input, as cost lowering reads it once; ids are per node /// pointer, so this dedupes by pointer. +/// +/// Every operator is listed: a new one fails to build here until its input +/// roles are decided, rather than getting one `Input`. +#[warn(clippy::wildcard_enum_match_arm)] fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { use EdgeRole::*; let inputs: Vec = match operator { @@ -578,12 +582,31 @@ fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> NonASAPOp::Scan { .. } | NonASAPOp::Values { .. } | NonASAPOp::PromqlVectorFromScalar(_) => vec![], - _ => vec![Input], + NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. } => vec![Input], }, Operator::ASAP(op) => match op { ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], - _ => vec![Input], + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::Extension { .. } => vec![Input], }, }; let children = operator.children(); diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index a9ab9a4cf..f6c960fc0 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -23,6 +23,20 @@ //! read through `TimeWindow { ends_before_evaluation_ms: 60_000, //! length_ms: 300_000 }`, `(t - 6m, t - 1m]`, reaching back 360,000 ms. +// The rules here answer per operator, and each lists every operator: a +// wildcard arm would give a new operator a default answer instead of +// failing the build (under `-D warnings`). Clippy does not check +// `matches!` and `let ... else`, so they stay only for a check of one +// variant's identity, where every other operator's answer is "no" by +// definition: "is it a `Scan`" (only a raw scan's read at t needs every +// row), "is it a `TimeShift`" (the shifts directly below a window), and +// the lifetime check "is it stored across evaluations" in +// `InputTimeAccess::of` (`matches!(lifetime, Lifetime::AcrossEvaluations(_))`). +#![warn( + clippy::wildcard_enum_match_arm, + clippy::match_wildcard_for_single_variants +)] + use serde::{Deserialize, Serialize}; use crate::ir::operator::asap::ASAPOp; @@ -128,6 +142,17 @@ impl InputTimeAccess { reads, shifts_back_ms: 0, }; + // The window a `TimeRange` or a chain covers, read in one place + // ([`Operator::covered_window_ms`]); only those two arms call this. + let last_window = || { + reads(InputRows::LastWindow { + length_ms: operator + .covered_window_ms() + .expect("a TimeRange or a tumbling-window chain covers a window"), + }) + }; + // Every operator is listed, so a new one fails to build here until + // its input time access is decided. match operator { Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { None => Self { @@ -141,19 +166,47 @@ impl InputTimeAccess { }, Some(_) => reads(InputRows::AllRows), }, - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => reads(InputRows::LastWindow { - length_ms: u64::try_from(range.as_millis()).unwrap_or(u64::MAX), - }), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => match timing { - ExecutionTiming::IngestionTime => reads(InputRows::LastWindow { - length_ms: *tumbling_window_width_ms, - }), + Operator::NonASAP(NonASAPOp::TimeRange { .. }) => last_window(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => match timing { + ExecutionTiming::IngestionTime => last_window(), ExecutionTiming::QueryTime => reads(InputRows::WhatItsReadersRead), }, - _ => reads(InputRows::CurrentTimeOnly), + // A subquery evaluates its input at other times than its own, + // but is read here as the rows at t. + Operator::NonASAP(NonASAPOp::PromqlSubquery { .. }) => { + reads(InputRows::CurrentTimeOnly) + } + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => reads(InputRows::CurrentTimeOnly), } } } @@ -308,14 +361,13 @@ pub fn shift_back_ms_below(plan: &P, node: P::Node) -> Optio // shift. let mut path_ends = inputs.is_empty(); for input in inputs { - match plan.operator_of(input) { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - if shift.at.is_some() { - return None; - } - pending.push((input, walked_ms.checked_add(shift.offset_ms)?)); + if let Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) = plan.operator_of(input) { + if shift.at.is_some() { + return None; } - _ => path_ends = true, + pending.push((input, walked_ms.checked_add(shift.offset_ms)?)); + } else { + path_ends = true; } } if path_ends { @@ -347,18 +399,9 @@ pub fn windows_computed_from( if !seen.insert(id) { continue; } - let length_ms = match plan.operator_of(id) { - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => { - u64::try_from(range.as_millis()).ok()? - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => *tumbling_window_width_ms, - _ => { - pending.extend(plan.inputs_of(id)); - continue; - } + let Some(length_ms) = plan.operator_of(id).covered_window_ms() else { + pending.extend(plan.inputs_of(id)); + continue; }; windows.push(TimeWindow { ends_before_evaluation_ms: shift_back_ms_below(plan, id)?, @@ -395,6 +438,117 @@ mod tests { assert_eq!(shifted.bounds_ms(1_000_000), (640_000, 940_000)); } + /// The operators whose input time access is a decision, not the rows + /// at t: `TimeShift` by timing and lifetime, `@ end`, `TimeRange` and + /// the chain, plus `PromqlSubquery`, which keeps reading the rows at t + /// although it evaluates its input at other times. Pins the operators + /// that exist today; a new operator fails to build in + /// [`InputTimeAccess::of`], and its decision gets a row here. + #[test] + fn operators_with_a_time_access_decision() { + use crate::ir::operator::{AggIntent, AtModifier, Reduction, TimeShift}; + use crate::ir::scalar::ColumnRef; + use crate::ir::schema::FieldDataType; + use crate::ir::TimeRangeKind; + use crate::stages::StorageMedium; + use crate::summary_type::{SketchAlgorithm, SummaryUpdate}; + use std::time::Duration; + use ExecutionTiming::{IngestionTime, QueryTime}; + use InputRows::*; + + let non_asap = |op: NonASAPOp| Operator::NonASAP(op); + let shift = |at| { + non_asap(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at, + }, + child: 0, + }) + }; + let chain = || { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }) + }; + let subquery = non_asap(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(300), + resolution: None, + child: 0, + }); + let aggregate = non_asap(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Rate], + output_names: vec![String::new()], + filters: vec![], + having: None, + child: 0, + }); + let stored = Lifetime::AcrossEvaluations(StorageMedium::Memory); + let once = Lifetime::OneEvaluation(StorageMedium::Memory); + let access = |reads, shifts_back_ms| InputTimeAccess { + reads, + shifts_back_ms, + }; + let table: Vec<(Operator, ExecutionTiming, Lifetime, InputTimeAccess)> = vec![ + ( + shift(None), + QueryTime, + once, + access(WhatItsReadersRead, 3_600_000), + ), + ( + shift(None), + IngestionTime, + once, + access(WhatItsReadersRead, 3_600_000), + ), + ( + shift(None), + IngestionTime, + stored, + access(CurrentTimeOnly, 3_600_000), + ), + ( + shift(Some(AtModifier::End)), + QueryTime, + once, + access(AllRows, 0), + ), + ( + non_asap(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: 0, + }), + QueryTime, + once, + access(LastWindow { length_ms: 300_000 }, 0), + ), + (chain(), QueryTime, once, access(WhatItsReadersRead, 0)), + ( + chain(), + IngestionTime, + stored, + access(LastWindow { length_ms: 60_000 }, 0), + ), + (subquery, QueryTime, once, access(CurrentTimeOnly, 0)), + (aggregate, IngestionTime, stored, access(CurrentTimeOnly, 0)), + ]; + for (operator, timing, lifetime, expected) in table { + assert_eq!( + InputTimeAccess::of(&operator, timing, lifetime), + expected, + "{operator:?} at {timing:?}, {lifetime:?}" + ); + } + } + /// A hand-built plan: each node's readers and input time access, and /// whether node 0 is raw data. The walks up read only these. struct Plan { diff --git a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json index a5bcc3476..d6cfcf314 100644 --- a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json +++ b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} diff --git a/tools/dag-viewer/test-fixtures/memo-small.json b/tools/dag-viewer/test-fixtures/memo-small.json index 2b8af0eb1..8f87de68c 100644 --- a/tools/dag-viewer/test-fixtures/memo-small.json +++ b/tools/dag-viewer/test-fixtures/memo-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}}