From 202855b7656ad70cba2027221407766616f2704a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:18:29 +0000 Subject: [PATCH 1/6] feat(types, cost): price DDSketch state from declared column_magnitudes SourceFacts gains column_magnitudes (ValueMagnitude {min_abs, max_abs} per column). A DDSketch state built from a column its producer passes on unchanged, from sources that all declare it, is priced at the bins that range spans instead of the assumed 1e9 range. Invalid magnitudes are a WorkloadError::InvalidValueMagnitude naming source and column. --- crates/cost/src/node/state_bytes/mod.rs | 54 +++++++---- crates/cost/src/node/state_bytes/tests.rs | 41 +++++++- crates/cost/src/node/usage/mod.rs | 42 ++++++-- crates/cost/src/node/usage/tests.rs | 95 +++++++++++++++++++ crates/cost/src/plan/cost.rs | 2 +- crates/types/src/workload/bounds.rs | 62 +++++++++++- crates/types/src/workload/data.rs | 77 +++++++++++++++ crates/types/src/workload/mod.rs | 2 +- crates/types/src/workload/planning.rs | 23 +++++ .../architecture/input-output-workflow.md | 6 ++ docs/develop_docs/library-api.md | 5 +- 11 files changed, 380 insertions(+), 29 deletions(-) diff --git a/crates/cost/src/node/state_bytes/mod.rs b/crates/cost/src/node/state_bytes/mod.rs index 6f039cfea..c1e851cb6 100644 --- a/crates/cost/src/node/state_bytes/mod.rs +++ b/crates/cost/src/node/state_bytes/mod.rs @@ -6,6 +6,7 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::summary_type::KLL_MAX_K; use asap_types::summary_type::{SketchParams, EXACT_ACCUMULATOR_STATE_BYTES}; +use asap_types::workload::ValueMagnitude; /// Plain values are 8 bytes, strings 16; summary columns are sized apart. pub fn row_bytes(schema: &Schema) -> u64 { @@ -43,7 +44,9 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// `ASSUMED_VALUE_KEY_BYTES`; /// - DDSketch and KMV: the rows the state folds, at least one (any number /// when unknown), each a distinct item for KMV; a DDSketch's values span -/// `DDSKETCH_ASSUMED_VALUE_RANGE` and fill its negative store too +/// `value_magnitude` when declared (its bins are `ceil(log_γ(max_abs / +/// min_abs))` for its α; this DDSketch has no bin limit to cap them), +/// else `DDSKETCH_ASSUMED_VALUE_RANGE`, and fill its negative store too /// unless proven non-negative. /// /// KLL is priced without the library's estimate: its fixed bytes @@ -66,9 +69,12 @@ pub fn summary_shape( family: &FieldDataType, parameters: Option<&SketchParams>, input_rows: StateInputRows, + value_magnitude: Option, ) -> Result<(f64, u64), AnalyticalCostError> { Ok(match (family, parameters) { - (FieldDataType::Sketch(..), Some(parameters)) => sketch_shape(parameters, input_rows), + (FieldDataType::Sketch(..), Some(parameters)) => { + sketch_shape(parameters, input_rows, value_magnitude) + } (FieldDataType::Sketch(..), None) => return Err(AnalyticalCostError::UnsupportedCandidate), (FieldDataType::ExactAggregate(..), _) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), _ => (1.0, 8), @@ -100,10 +106,14 @@ impl StateInputRows { } /// [`summary_shape`] of one sketch: its update operations per row and its -/// bytes per state. With [`StateInputRows::UNKNOWN`], the bytes are also the -/// size the measured table interpolates between -/// (`crate::measured`). -pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, u64) { +/// bytes per state. With [`StateInputRows::UNKNOWN`] and no +/// `value_magnitude`, the bytes are also the size the measured table +/// interpolates between (`crate::measured`). +pub fn sketch_shape( + params: &SketchParams, + input_rows: StateInputRows, + value_magnitude: Option, +) -> (f64, u64) { use asap_sketchlib::{ Classic, CountMinF64, CountMinSketchWithHeap, CountSketch, CountSketchWithHeap, DataInput, DdSketch, HyperLogLog, Kmv, RegularPath, UnivMon, Vector2D, @@ -113,7 +123,9 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, usize::try_from(rows.max(1)).unwrap_or(usize::MAX) }); match params { - SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), + SketchParams::Hydra(hydra) => { + sketch_shape(&hydra.per_subpopulation(), input_rows, value_magnitude) + } SketchParams::Cms { width, depth } => ( f64::from(*depth), CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, @@ -160,16 +172,22 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, SketchParams::DDSketch { alpha } if !(*alpha > 0.0 && *alpha < 1.0) => { (1.0, std::mem::size_of::() as u64) } - SketchParams::DDSketch { alpha } => ( - 1.0, - DdSketch::estimated_memory_bytes( - *alpha, - rows, + SketchParams::DDSketch { alpha } => { + let ValueMagnitude { min_abs, max_abs } = value_magnitude.unwrap_or(ValueMagnitude { + min_abs: 1.0, + max_abs: DDSKETCH_ASSUMED_VALUE_RANGE, + }); + ( 1.0, - DDSKETCH_ASSUMED_VALUE_RANGE, - !input_rows.non_negative, - ) as u64, - ), + DdSketch::estimated_memory_bytes( + *alpha, + rows, + min_abs, + max_abs, + !input_rows.non_negative, + ) as u64, + ) + } SketchParams::Hll { precision, .. } => ( 1.0, HyperLogLog::::estimated_memory_bytes(usize::from(*precision)) as u64, @@ -273,8 +291,8 @@ fn univmon_expected_layers(layers: u8) -> f64 { /// Ratio of the largest to the smallest value magnitude a DDSketch is /// assumed to see: nine decades, e.g. latencies from 1 ns to 1 s, priced as /// magnitudes in `[1, 1e9]` (its bins depend on the ratio alone). An -/// assumption, not workload evidence: the data workload declares no value -/// range. +/// assumption, not workload evidence, for a column whose source declares +/// no `column_magnitudes`. const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; #[cfg(test)] diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index add5ba6fe..a50da8b50 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -16,7 +16,8 @@ fn exact_state_is_priced_at_the_executors_state_size() { summary_shape( &FieldDataType::ExactAggregate(kind, params), None, - StateInputRows::UNKNOWN + StateInputRows::UNKNOWN, + None ), Ok((1.0, 56)) ); @@ -109,6 +110,7 @@ fn ddsketch_state_bytes_bound_the_library() { rows_per_state, non_negative, }, + None, ) .1 }; @@ -145,6 +147,41 @@ fn ddsketch_state_bytes_bound_the_library() { assert_eq!(priced(Some(10_000), false), priced(None, false)); } +/// A DDSketch over a column declaring its value magnitudes is priced at +/// the bins they span, not nine decades. At α = 0.01, γ = 1.01 / 0.99 and +/// a value `v` falls in bin `floor(ln v / ln γ)`: magnitudes in +/// [0.001, 10000] take bins -346 to 460, 807 bins, where the default +/// [1, 1e9] takes 0 to 1036, 1,037. With three or more non-negative rows +/// the library's bound is `2 (bins + 126)` 8-byte slots plus the struct. +#[test] +fn ddsketch_state_bytes_follow_the_declared_value_magnitude() { + let alpha = 0.01; + let priced = |value_magnitude| { + sketch_shape( + &SketchParams::DDSketch { alpha }, + StateInputRows { + rows_per_state: Some(600), + non_negative: true, + }, + value_magnitude, + ) + .1 + }; + let struct_bytes = std::mem::size_of::() as u64; + let bytes_for_bins = |bins: u64| struct_bytes + 8 * 2 * (bins + 126); + let latency = ValueMagnitude { + min_abs: 0.001, + max_abs: 10_000.0, + }; + assert_eq!(priced(Some(latency)), bytes_for_bins(807)); + assert_eq!(priced(None), bytes_for_bins(1_037)); + // Values a narrower range holds fit the priced bytes. + let values: Vec = (0..600) + .map(|i| 10f64.powf(-3.0 + 7.0 * f64::from(i) / 599.0)) + .collect(); + assert!(library_ddsketch_bytes(alpha, &values) <= priced(Some(latency))); +} + /// A KLL state is charged what `asap_sketchlib` allocates for the /// samples it sees, not its full capacity: its fixed struct and level /// array, plus the item array the library starts empty and doubles from @@ -169,6 +206,7 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { rows_per_state, non_negative: true, }, + None, ) .1 }; @@ -385,6 +423,7 @@ fn state_bytes_bound_the_executors_charge() { rows_per_state: Some(u64::from(n)), non_negative: false, }, + None, ) .1 as usize; let context = format!("{algorithm:?} after {n} rows: {priced} priced"); diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 5288d954f..7fa5508d4 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -35,16 +35,17 @@ use crate::{ }; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::resolve_column_ref; use asap_types::ir::schema::{FieldDataType, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; use asap_types::stages::{Lifetime, StorageMedium}; -use asap_types::summary_type::{SketchParams, SketchStatistic}; +use asap_types::summary_type::{SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate}; use asap_types::workload::bounds::{ - declared_non_negative, declared_series, group_count_bound, source_name, source_rows_per_ms, - unchanged_source_columns, + declared_non_negative, declared_series, declared_value_magnitude, group_count_bound, + source_name, source_rows_per_ms, unchanged_source_columns, }; use asap_types::workload::{DataWorkload, ONE_SECOND_MS}; @@ -314,7 +315,7 @@ pub fn node_resource_usage_and_cost<'a>( let speed = settings.resource_speed; let unit_costs = settings.cost_model.per_node_unit_costs(); let sketch_size = - |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; + |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN, None).1 as f64; let mut fit = CalibrationFit::default(); // Query-time memory held while the node computes. let mut held_bytes = None; @@ -494,6 +495,23 @@ pub fn node_resource_usage_and_cost<'a>( .iter() .all(|source| declared_non_negative(data, source)) }; + // The declared magnitudes of the values a summary built here folds: + // its `weight` column, when the producer passes it on unchanged from + // the sources and every one of them declares its `column_magnitudes`. + // E.g. a DDSketch over a scan of `latency` declaring `value` in + // [0.001, 10000]. + let folded_value_magnitude = |update: &SummaryUpdate| { + let SummaryInputExpr::Column(column) = &update.weight else { + return None; + }; + let producer = producers.first()?; + let index = resolve_column_ref(column, producer.output_schema).ok()?; + let name = producer.output_schema.fields.get(index)?.name.as_str(); + if !producer.unchanged_columns.contains(name) { + return None; + } + declared_value_magnitude(data, sources.iter().copied(), name) + }; let unary = |output| UnaryEdgeStatistics { input, output, @@ -807,6 +825,7 @@ pub fn node_resource_usage_and_cost<'a>( }, Payload::ASAP(ASAPOp::SummaryAgg { summary_type: family, + input: update, reduction, .. }) => { @@ -818,7 +837,12 @@ pub fn node_resource_usage_and_cost<'a>( rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), non_negative: raw_data_samples_non_negative(), }; - let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; + let (depth, state_bytes) = summary_shape( + family, + node.parameters.as_ref(), + input_rows, + folded_value_magnitude(update), + )?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); let out = EdgeStatistics { rows: group_count, @@ -842,6 +866,7 @@ pub fn node_resource_usage_and_cost<'a>( } Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type: family, + input: update, reduction, tumbling_window_width_ms, .. @@ -863,7 +888,12 @@ pub fn node_resource_usage_and_cost<'a>( rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), non_negative: raw_data_samples_non_negative(), }; - let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; + let (depth, state_bytes) = summary_shape( + family, + node.parameters.as_ref(), + input_rows, + folded_value_magnitude(update), + )?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); let out = EdgeStatistics { rows: group_count, diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 98fadfad6..b7241492f 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -486,3 +486,98 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { .unwrap(); assert_eq!(usage.peak_memory_bytes, 16_000 * 66); } + +/// A DDSketch built over a scan's `value` is priced at the bins its +/// source's declared `column_magnitudes` span: 807 at α = 0.01 for +/// [0.001, 10000], against 1,037 for the assumed [1, 1e9]. Undeclared, or +/// a column its producer does not pass on unchanged, keeps the assumed +/// range. +#[test] +fn a_ddsketch_over_a_scan_is_priced_at_its_declared_value_magnitude() { + use crate::node::state_bytes::{sketch_shape, StateInputRows}; + use asap_types::ir::schema::{DataType, Field}; + use asap_types::summary_type::SketchAlgorithm; + use asap_types::workload::{SourceFacts, ValueMagnitude}; + let settings = CostInputs::default(); + let alpha = 0.01; + let latency = ValueMagnitude { + min_abs: 0.001, + max_abs: 10_000.0, + }; + let data = |magnitude: Option| DataWorkload { + sources: BTreeMap::from([( + "latency".to_string(), + SourceFacts { + column_magnitudes: magnitude + .map(|magnitude| ("value".to_string(), magnitude)) + .into_iter() + .collect(), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + let scan = PhysicalASAPDAGNode { + output_schema: Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]), + ..node(Payload::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::default(), + })) + }; + let build = PhysicalASAPDAGNode { + parameters: Some(SketchParams::DDSketch { alpha }), + ..node(Payload::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::Reduce(GroupKeys::default()), + filter: None, + })) + }; + let state_bytes = |data: &DataWorkload, unchanged_columns: BTreeSet<&str>| { + let scan_output = ProducerOutput { + payload: &scan.payload, + output_schema: &scan.output_schema, + output: EdgeStatistics { + rows: 600, + bytes: 9_600, + }, + sources: BTreeSet::from(["latency"]), + family: StateFamily::None, + unchanged_columns, + tumbling_windows: 1, + }; + node_resource_usage_and_cost(&build, &[scan_output], (1, 1.0), None, data, &settings) + .unwrap() + .output + .bytes + }; + let priced = |magnitude| { + sketch_shape( + &SketchParams::DDSketch { alpha }, + StateInputRows { + rows_per_state: Some(600), + non_negative: false, + }, + magnitude, + ) + .1 + }; + let unchanged = || BTreeSet::from(["ts", "value"]); + assert_eq!( + state_bytes(&data(Some(latency)), unchanged()), + priced(Some(latency)) + ); + assert_eq!(state_bytes(&data(None), unchanged()), priced(None)); + assert_eq!( + state_bytes(&data(Some(latency)), BTreeSet::from(["ts"])), + priced(None) + ); + assert!(priced(Some(latency)) < priced(None)); +} diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 41253cf88..d4cf1c36c 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -190,7 +190,7 @@ pub fn plan_cost( // finalizations over it. let mut family_of: HashMap> = HashMap::new(); let sketch_size = - |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; + |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN, None).1 as f64; for node in &dag.nodes { let node_costed = &costed[&node.id]; usage.cpu_ops_per_second += node_costed.cpu_ops_per_second; diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 4b17f2150..eebdbe0af 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -12,7 +12,7 @@ use crate::ir::{ }; use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; use crate::summary_type::SketchStatistic; -use crate::workload::{DataWorkload, PrometheusMetricType}; +use crate::workload::{DataWorkload, PrometheusMetricType, ValueMagnitude}; /// A scan source's name: its PromQL metric or SQL table. pub fn source_name(source: &Source) -> &str { @@ -415,6 +415,32 @@ pub fn declared_non_negative(data: &DataWorkload, metric: &str) -> bool { }) } +/// The magnitudes of `column`'s non-zero values over every source in +/// `sources`, from each one's declared `column_magnitudes` +/// ([`SourceFacts::column_magnitudes`](crate::workload::SourceFacts::column_magnitudes)): +/// the smallest `min_abs` and the largest `max_abs` among them. `None` when +/// `sources` is empty or some source declares none for `column`. E.g. +/// sources declaring `[0.001, 10]` and `[0.01, 100]` give `[0.001, 100]`. +pub fn declared_value_magnitude<'a>( + data: &DataWorkload, + sources: impl IntoIterator, + column: &str, +) -> Option { + sources + .into_iter() + .try_fold(None, |union: Option, source| { + let (facts, _) = data.series_facts(source)?; + let declared = *facts.column_magnitudes.get(column)?; + Some(Some(match union { + None => declared, + Some(union) => ValueMagnitude { + min_abs: union.min_abs.min(declared.min_abs), + max_abs: union.max_abs.max(declared.max_abs), + }, + })) + })? +} + /// Whether `data` declares `metric` a counter, itself or as the `_sum` or /// `_count` series of a declared histogram or summary /// ([`DataWorkload::series_facts`]). A declared type does not expire, so its @@ -679,6 +705,40 @@ mod tests { use super::*; use crate::workload::{DataArrival, DurationMs, SourceFacts}; + /// The magnitudes over several sources span all of theirs, and are + /// unknown when any of them declares none for the column. + #[test] + fn declared_value_magnitudes_are_the_union_over_the_sources() { + let facts = |min_abs, max_abs| SourceFacts { + column_magnitudes: BTreeMap::from([( + "value".to_string(), + ValueMagnitude { min_abs, max_abs }, + )]), + ..SourceFacts::default() + }; + let data = DataWorkload { + sources: BTreeMap::from([ + ("a".to_string(), facts(0.001, 10.0)), + ("b".to_string(), facts(0.01, 100.0)), + ("c".to_string(), SourceFacts::default()), + ]), + ..DataWorkload::default() + }; + let magnitude = |sources: &[&str], column| { + declared_value_magnitude(&data, sources.iter().copied(), column) + }; + assert_eq!( + magnitude(&["a", "b"], "value"), + Some(ValueMagnitude { + min_abs: 0.001, + max_abs: 100.0 + }) + ); + assert_eq!(magnitude(&["a", "c"], "value"), None); + assert_eq!(magnitude(&["a"], "latency"), None); + assert_eq!(magnitude(&[], "value"), None); + } + /// A source's rows per ms: its series over its sample interval when it /// declares both, else its series share of the workload's rate. #[test] diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 93385c36e..cb7d3356f 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -126,6 +126,15 @@ pub struct SourceFacts { /// Omitted from JSON when empty. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub column_signs: BTreeMap, + /// Bounds on the magnitude of each column's non-zero values, keyed by + /// column name (default: none declared), e.g. `{"value": {"min_abs": + /// 0.001, "max_abs": 10000.0}}` for a latency in seconds. A DDSketch + /// state built from the column unchanged is priced at the + /// `log_γ(max_abs / min_abs)` bins those magnitudes span instead of an + /// assumed range of nine decades. The deployment must ensure it. + /// Omitted from JSON when empty. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub column_magnitudes: BTreeMap, /// How often each series of the metric (or group of rows of a SQL /// table) receives a sample (default: unknown). A PromQL or MetricsQL /// bare instant selector over the metric reads the latest sample within it; lowering such a @@ -196,6 +205,19 @@ pub struct SourceFacts { pub rows_held: Option, } +/// Bounds on the magnitude of one column's non-zero values: +/// `min_abs <= |v| <= max_abs` for every non-zero `v`, with +/// `0 < min_abs <= max_abs`, both finite. E.g. `{"min_abs": 0.001, +/// "max_abs": 10000.0}` for latencies from 1 ms to about 3 h in seconds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ValueMagnitude { + /// The smallest magnitude of a non-zero value. + pub min_abs: f64, + /// The largest magnitude of a value. + pub max_abs: f64, +} + /// The distinct combinations of values a set of labels (or columns) of one /// source takes together, an upper bound. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -372,6 +394,21 @@ impl DataWorkload { } } } + for (column, magnitude) in &facts.column_magnitudes { + let ValueMagnitude { min_abs, max_abs } = *magnitude; + if !(min_abs.is_finite() + && max_abs.is_finite() + && 0.0 < min_abs + && min_abs <= max_abs) + { + return Err(WorkloadError::InvalidValueMagnitude { + source_name: source_name.clone(), + column: column.clone(), + min_abs, + max_abs, + }); + } + } if facts.max_distinct_per_evaluation == Some(0) { return Err(WorkloadError::ZeroMaxDistinct); } @@ -507,6 +544,46 @@ mod tests { assert_eq!(with(0).validate(), Err(WorkloadError::ZeroMaxDistinct)); } + /// A declared value magnitude has `0 < min_abs <= max_abs`, both + /// finite; any other is an input error naming the source and column. + #[test] + fn column_magnitudes_must_be_positive_ordered_and_finite() { + let with = |min_abs: f64, max_abs: f64| DataWorkload { + sources: BTreeMap::from([( + "latency".to_string(), + SourceFacts { + column_magnitudes: BTreeMap::from([( + "value".to_string(), + ValueMagnitude { min_abs, max_abs }, + )]), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(0.001, 10_000.0).validate().is_ok()); + assert!(with(5.0, 5.0).validate().is_ok()); + for (min_abs, max_abs) in [ + (0.0, 1.0), + (-1.0, 1.0), + (10.0, 1.0), + (1.0, f64::INFINITY), + (f64::NAN, 1.0), + (1.0, f64::NAN), + ] { + let error = with(min_abs, max_abs).validate().unwrap_err(); + assert!( + matches!( + &error, + WorkloadError::InvalidValueMagnitude { source_name, column, .. } + if source_name == "latency" && column == "value" + ), + "{min_abs}, {max_abs}: {error:?}" + ); + assert!(error.to_string().contains("min_abs"), "{error}"); + } + } + /// A declared `rows_held` is positive. #[test] fn rows_held_must_be_positive() { diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index cff20a85c..ab90b157c 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -11,7 +11,7 @@ mod query; pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, - StreamNorms, ONE_SECOND_MS, + StreamNorms, ValueMagnitude, ONE_SECOND_MS, }; pub use planning::{PlanningWorkload, WorkloadError}; pub use query::{ diff --git a/crates/types/src/workload/planning.rs b/crates/types/src/workload/planning.rs index 4ab90a894..290031af9 100644 --- a/crates/types/src/workload/planning.rs +++ b/crates/types/src/workload/planning.rs @@ -134,6 +134,18 @@ pub enum WorkloadError { distinct_values: u64, label_product: u64, }, + /// Bad input: a `column_magnitudes` entry that is not + /// `0 < min_abs <= max_abs` with both finite. + #[error( + "source {source_name}: column_magnitudes for column {column} must have \ + 0 < min_abs <= max_abs, both finite, got min_abs {min_abs}, max_abs {max_abs}" + )] + InvalidValueMagnitude { + source_name: String, + column: String, + min_abs: f64, + max_abs: f64, + }, /// Bad input: a source's `rows_held` of zero. #[error("a source's rows_held must be greater than zero")] ZeroRowsHeld, @@ -187,6 +199,13 @@ mod tests { "value".to_string(), ValueSign::NonNegative, )]), + column_magnitudes: BTreeMap::from([( + "value".to_string(), + ValueMagnitude { + min_abs: 1.0, + max_abs: 1e12, + }, + )]), sample_interval: Some(DurationMs(15_000)), stream_norms: Some(StreamNorms { l1: Some(1e9), @@ -216,6 +235,7 @@ mod tests { "series_count": 10000, "metric_type": "counter", "column_signs": {"value": "non_negative"}, + "column_magnitudes": {"value": {"min_abs": 1.0, "max_abs": 1e12}}, "sample_interval": 15000, "stream_norms": {"l1": 1e9, "l2": null}, "max_distinct_per_evaluation": 100, @@ -247,5 +267,8 @@ mod tests { ..SourceFacts::default() } ); + // Empty column magnitudes are omitted from JSON. + let json = serde_json::to_value(&minimal.sources["flows"]).unwrap(); + assert!(json.get("column_magnitudes").is_none(), "{json}"); } } diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 16348633b..dbbd43313 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -197,6 +197,7 @@ struct SourceFacts { series_count: Option, metric_type: Option, column_signs: BTreeMap, + column_magnitudes: BTreeMap, sample_interval: Option, stream_norms: Option, max_distinct_per_evaluation: Option, @@ -204,6 +205,10 @@ struct SourceFacts { distinct_values_per_label_set: Vec, rows_held: Option, } +struct ValueMagnitude { + min_abs: f64, + max_abs: f64, +} struct LabelSetDistinctValues { labels: Vec, distinct_values: u64, @@ -217,6 +222,7 @@ struct LabelSetDistinctValues { | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | | `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | | `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | +| `column_magnitudes` | Optional; default none; each `0 < min_abs <= max_abs`, both finite | Bounds on the magnitude of each column's non-zero values, keyed by column name, e.g. `{"value": {"min_abs": 0.001, "max_abs": 10000.0}}` for a latency in seconds. A DDSketch state built from a column its producer passes on unchanged from sources that all declare it is priced at the `log_γ(max_abs / min_abs)` bins that range spans (over several sources, the widest range among them), instead of an assumed range of nine decades. The deployment must ensure it. Any other value is an input error naming the source and column. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 126c6615d..658b3c214 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -306,7 +306,10 @@ query's window comes from its text (range and offset). These facts are separate: `DataWorkload` holds `arrival` (`DataArrival`: `AtRest`, `Ingesting` or `Mixed` with `rows_per_second`, or `Unknown`) and `sources`, a map from PromQL metric or SQL table name to `SourceFacts`: `series_count`, -`metric_type`, `column_signs`, `sample_interval` (PromQL and +`metric_type`, `column_signs`, `column_magnitudes` (bounds on each +column's non-zero value magnitudes, `{"value": {"min_abs": 0.001, +"max_abs": 10000.0}}`, which price a DDSketch state at the bins that range +spans instead of nine decades), `sample_interval` (PromQL and MetricsQL), `stream_norms`, `max_distinct_per_evaluation` and `distinct_values_per_label` (distinct values per label or column, which plan selection prices a `by (...)` grouping's groups at: the product of its From 7752cf6668cad11356976ccab886e40ee600723c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:28:35 +0000 Subject: [PATCH 2/6] refactor(cost): carry the value magnitude on StateInputRows StateInputRows gains value_magnitude next to non_negative, filled for SummaryAgg and SummaryTumblingWindowChain from the declared column magnitudes. sketch_shape and summary_shape keep main's signatures. --- crates/cost/src/node/state_bytes/mod.rs | 45 +++++++++++------------ crates/cost/src/node/state_bytes/tests.rs | 11 +++--- crates/cost/src/node/usage/mod.rs | 18 +++------ crates/cost/src/node/usage/tests.rs | 2 +- crates/cost/src/plan/cost.rs | 2 +- 5 files changed, 34 insertions(+), 44 deletions(-) diff --git a/crates/cost/src/node/state_bytes/mod.rs b/crates/cost/src/node/state_bytes/mod.rs index c1e851cb6..0e656b478 100644 --- a/crates/cost/src/node/state_bytes/mod.rs +++ b/crates/cost/src/node/state_bytes/mod.rs @@ -44,8 +44,9 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// `ASSUMED_VALUE_KEY_BYTES`; /// - DDSketch and KMV: the rows the state folds, at least one (any number /// when unknown), each a distinct item for KMV; a DDSketch's values span -/// `value_magnitude` when declared (its bins are `ceil(log_γ(max_abs / -/// min_abs))` for its α; this DDSketch has no bin limit to cap them), +/// `input_rows.value_magnitude` when declared (its bins are +/// `ceil(log_γ(max_abs / min_abs))` for its α; this DDSketch has no bin +/// limit to cap them), /// else `DDSKETCH_ASSUMED_VALUE_RANGE`, and fill its negative store too /// unless proven non-negative. /// @@ -69,12 +70,9 @@ pub fn summary_shape( family: &FieldDataType, parameters: Option<&SketchParams>, input_rows: StateInputRows, - value_magnitude: Option, ) -> Result<(f64, u64), AnalyticalCostError> { Ok(match (family, parameters) { - (FieldDataType::Sketch(..), Some(parameters)) => { - sketch_shape(parameters, input_rows, value_magnitude) - } + (FieldDataType::Sketch(..), Some(parameters)) => sketch_shape(parameters, input_rows), (FieldDataType::Sketch(..), None) => return Err(AnalyticalCostError::UnsupportedCandidate), (FieldDataType::ExactAggregate(..), _) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), _ => (1.0, 8), @@ -82,8 +80,8 @@ pub fn summary_shape( } /// What one summary state's bytes depend on besides its parameters (only -/// a KLL's, a DDSketch's and a KMV's do): the input rows it folds and the -/// sign of their values. +/// a KLL's, a DDSketch's and a KMV's do): the input rows it folds, the +/// sign of their values and, for a DDSketch, their magnitudes. /// The rows are whatever its producer emits (raw data samples, rates, sums), /// not a summary sample. #[derive(Debug, Clone, Copy, PartialEq)] @@ -95,6 +93,11 @@ pub struct StateInputRows { /// Whether every value folded is proven non-negative (raw data samples of /// sources declared non-negative), so a DDSketch fills no negative store. pub non_negative: bool, + /// Bounds on `|value|` of the non-zero values folded into the state; + /// `None` when not declared, which prices a DDSketch over the assumed + /// `DDSKETCH_ASSUMED_VALUE_RANGE`. E.g. `Some(ValueMagnitude { min_abs: + /// 0.001, max_abs: 10000.0 })` for a latency in seconds. + pub value_magnitude: Option, } impl StateInputRows { @@ -102,18 +105,15 @@ impl StateInputRows { pub const UNKNOWN: Self = Self { rows_per_state: None, non_negative: false, + value_magnitude: None, }; } /// [`summary_shape`] of one sketch: its update operations per row and its -/// bytes per state. With [`StateInputRows::UNKNOWN`] and no -/// `value_magnitude`, the bytes are also the size the measured table -/// interpolates between (`crate::measured`). -pub fn sketch_shape( - params: &SketchParams, - input_rows: StateInputRows, - value_magnitude: Option, -) -> (f64, u64) { +/// bytes per state. With [`StateInputRows::UNKNOWN`], the bytes are also the +/// size the measured table interpolates between +/// (`crate::measured`). +pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, u64) { use asap_sketchlib::{ Classic, CountMinF64, CountMinSketchWithHeap, CountSketch, CountSketchWithHeap, DataInput, DdSketch, HyperLogLog, Kmv, RegularPath, UnivMon, Vector2D, @@ -123,9 +123,7 @@ pub fn sketch_shape( usize::try_from(rows.max(1)).unwrap_or(usize::MAX) }); match params { - SketchParams::Hydra(hydra) => { - sketch_shape(&hydra.per_subpopulation(), input_rows, value_magnitude) - } + SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), SketchParams::Cms { width, depth } => ( f64::from(*depth), CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, @@ -173,10 +171,11 @@ pub fn sketch_shape( (1.0, std::mem::size_of::() as u64) } SketchParams::DDSketch { alpha } => { - let ValueMagnitude { min_abs, max_abs } = value_magnitude.unwrap_or(ValueMagnitude { - min_abs: 1.0, - max_abs: DDSKETCH_ASSUMED_VALUE_RANGE, - }); + let ValueMagnitude { min_abs, max_abs } = + input_rows.value_magnitude.unwrap_or(ValueMagnitude { + min_abs: 1.0, + max_abs: DDSKETCH_ASSUMED_VALUE_RANGE, + }); ( 1.0, DdSketch::estimated_memory_bytes( diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index a50da8b50..546ed0d4a 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -16,8 +16,7 @@ fn exact_state_is_priced_at_the_executors_state_size() { summary_shape( &FieldDataType::ExactAggregate(kind, params), None, - StateInputRows::UNKNOWN, - None + StateInputRows::UNKNOWN ), Ok((1.0, 56)) ); @@ -109,8 +108,8 @@ fn ddsketch_state_bytes_bound_the_library() { StateInputRows { rows_per_state, non_negative, + value_magnitude: None, }, - None, ) .1 }; @@ -162,8 +161,8 @@ fn ddsketch_state_bytes_follow_the_declared_value_magnitude() { StateInputRows { rows_per_state: Some(600), non_negative: true, + value_magnitude, }, - value_magnitude, ) .1 }; @@ -205,8 +204,8 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { StateInputRows { rows_per_state, non_negative: true, + value_magnitude: None, }, - None, ) .1 }; @@ -422,8 +421,8 @@ fn state_bytes_bound_the_executors_charge() { StateInputRows { rows_per_state: Some(u64::from(n)), non_negative: false, + value_magnitude: None, }, - None, ) .1 as usize; let context = format!("{algorithm:?} after {n} rows: {priced} priced"); diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 7fa5508d4..bb2592668 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -315,7 +315,7 @@ pub fn node_resource_usage_and_cost<'a>( let speed = settings.resource_speed; let unit_costs = settings.cost_model.per_node_unit_costs(); let sketch_size = - |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN, None).1 as f64; + |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; let mut fit = CalibrationFit::default(); // Query-time memory held while the node computes. let mut held_bytes = None; @@ -836,13 +836,9 @@ pub fn node_resource_usage_and_cost<'a>( let input_rows = StateInputRows { rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), non_negative: raw_data_samples_non_negative(), + value_magnitude: folded_value_magnitude(update), }; - let (depth, state_bytes) = summary_shape( - family, - node.parameters.as_ref(), - input_rows, - folded_value_magnitude(update), - )?; + let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); let out = EdgeStatistics { rows: group_count, @@ -887,13 +883,9 @@ pub fn node_resource_usage_and_cost<'a>( let input_rows = StateInputRows { rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), non_negative: raw_data_samples_non_negative(), + value_magnitude: folded_value_magnitude(update), }; - let (depth, state_bytes) = summary_shape( - family, - node.parameters.as_ref(), - input_rows, - folded_value_magnitude(update), - )?; + let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); let out = EdgeStatistics { rows: group_count, diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index b7241492f..626091fa7 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -564,8 +564,8 @@ fn a_ddsketch_over_a_scan_is_priced_at_its_declared_value_magnitude() { StateInputRows { rows_per_state: Some(600), non_negative: false, + value_magnitude: magnitude, }, - magnitude, ) .1 }; diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index d4cf1c36c..41253cf88 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -190,7 +190,7 @@ pub fn plan_cost( // finalizations over it. let mut family_of: HashMap> = HashMap::new(); let sketch_size = - |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN, None).1 as f64; + |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; for node in &dag.nodes { let node_costed = &costed[&node.id]; usage.cpu_ops_per_second += node_costed.cpu_ops_per_second; From 8becfd4e676c8aee5d1028747587e8019c87f55d Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:54:14 +0000 Subject: [PATCH 3/6] fix(types, cost): read magnitudes per series; one folded-column rule for sign and range - Read `column_magnitudes` from the series' own entry, not its family's: a histogram family's range no longer bounds `foo_sum`, `foo_count` or `foo_bucket`. - One rule picks the folded source column (`folded_source_column`); both `non_negative` and `value_magnitude_range` come from it. `declared_non_negative` takes the column, keeping the counter case for `value`, so a SQL column declared non-negative and a filter over a non-negative scan now cost one DDSketch store. - Rename `ValueMagnitude` to `ValueMagnitudeRange` (and the error variant, the bounds function and the `StateInputRows` field); one `DDSKETCH_ASSUMED_MAGNITUDE` constant; simpler union fold. - Docs: exact bin count, per-series lookup, and the too-narrow-range warning; "cost" instead of "price" in the new comments. --- crates/cost/src/node/state_bytes/mod.rs | 50 ++-- crates/cost/src/node/state_bytes/tests.rs | 28 +- crates/cost/src/node/usage/mod.rs | 55 ++-- crates/cost/src/node/usage/tests.rs | 249 +++++++++++++----- .../tests/histogram_metadata.rs | 10 +- crates/types/src/workload/bounds.rs | 111 ++++++-- crates/types/src/workload/data.rs | 28 +- crates/types/src/workload/mod.rs | 2 +- crates/types/src/workload/planning.rs | 4 +- .../architecture/input-output-workflow.md | 8 +- docs/develop_docs/library-api.md | 4 +- 11 files changed, 360 insertions(+), 189 deletions(-) diff --git a/crates/cost/src/node/state_bytes/mod.rs b/crates/cost/src/node/state_bytes/mod.rs index 0e656b478..955fe3104 100644 --- a/crates/cost/src/node/state_bytes/mod.rs +++ b/crates/cost/src/node/state_bytes/mod.rs @@ -6,7 +6,7 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::summary_type::KLL_MAX_K; use asap_types::summary_type::{SketchParams, EXACT_ACCUMULATOR_STATE_BYTES}; -use asap_types::workload::ValueMagnitude; +use asap_types::workload::ValueMagnitudeRange; /// Plain values are 8 bytes, strings 16; summary columns are sized apart. pub fn row_bytes(schema: &Schema) -> u64 { @@ -44,11 +44,12 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// `ASSUMED_VALUE_KEY_BYTES`; /// - DDSketch and KMV: the rows the state folds, at least one (any number /// when unknown), each a distinct item for KMV; a DDSketch's values span -/// `input_rows.value_magnitude` when declared (its bins are -/// `ceil(log_γ(max_abs / min_abs))` for its α; this DDSketch has no bin -/// limit to cap them), -/// else `DDSKETCH_ASSUMED_VALUE_RANGE`, and fill its negative store too -/// unless proven non-negative. +/// `input_rows.value_magnitude_range` when declared, else +/// `DDSKETCH_ASSUMED_MAGNITUDE`, and fill its negative store too unless +/// proven non-negative. Its bins are the bucket indices from +/// ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one, for γ = (1 + α) / +/// (1 - α); this DDSketch has no bin limit to cap them. E.g. α = 0.01 +/// over [0.001, 10000] takes indices -346 to 460, 807 bins. /// /// KLL is priced without the library's estimate: its fixed bytes /// (`kll_fixed_bytes`: the struct and its level array) plus the 8-byte @@ -90,14 +91,15 @@ pub struct StateInputRows { /// `None` when unknown, which sizes it for every value it can be assumed /// to see. pub rows_per_state: Option, - /// Whether every value folded is proven non-negative (raw data samples of - /// sources declared non-negative), so a DDSketch fills no negative store. + /// Whether every value folded is proven non-negative (a source column + /// passed on unchanged, which every source declares non-negative), so a + /// DDSketch fills no negative store. pub non_negative: bool, /// Bounds on `|value|` of the non-zero values folded into the state; - /// `None` when not declared, which prices a DDSketch over the assumed - /// `DDSKETCH_ASSUMED_VALUE_RANGE`. E.g. `Some(ValueMagnitude { min_abs: - /// 0.001, max_abs: 10000.0 })` for a latency in seconds. - pub value_magnitude: Option, + /// `None` when not declared, which costs a DDSketch over the assumed + /// `DDSKETCH_ASSUMED_MAGNITUDE`. E.g. `Some(ValueMagnitudeRange { + /// min_abs: 0.001, max_abs: 10000.0 })` for a latency in seconds. + pub value_magnitude_range: Option, } impl StateInputRows { @@ -105,7 +107,7 @@ impl StateInputRows { pub const UNKNOWN: Self = Self { rows_per_state: None, non_negative: false, - value_magnitude: None, + value_magnitude_range: None, }; } @@ -171,11 +173,9 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, (1.0, std::mem::size_of::() as u64) } SketchParams::DDSketch { alpha } => { - let ValueMagnitude { min_abs, max_abs } = - input_rows.value_magnitude.unwrap_or(ValueMagnitude { - min_abs: 1.0, - max_abs: DDSKETCH_ASSUMED_VALUE_RANGE, - }); + let ValueMagnitudeRange { min_abs, max_abs } = input_rows + .value_magnitude_range + .unwrap_or(DDSKETCH_ASSUMED_MAGNITUDE); ( 1.0, DdSketch::estimated_memory_bytes( @@ -287,12 +287,14 @@ fn univmon_expected_layers(layers: u8) -> f64 { 2.0 - 2f64.powi(1 - i32::from(layers.max(1))) } -/// Ratio of the largest to the smallest value magnitude a DDSketch is -/// assumed to see: nine decades, e.g. latencies from 1 ns to 1 s, priced as -/// magnitudes in `[1, 1e9]` (its bins depend on the ratio alone). An -/// assumption, not workload evidence, for a column whose source declares -/// no `column_magnitudes`. -const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; +/// The value magnitudes a DDSketch is assumed to see when its column +/// declares no `column_magnitudes`: nine decades, e.g. latencies from 1 ns +/// to 1 s, costed as magnitudes in `[1, 1e9]` (its bins depend on the ratio +/// alone). An assumption, not workload evidence. +const DDSKETCH_ASSUMED_MAGNITUDE: ValueMagnitudeRange = ValueMagnitudeRange { + min_abs: 1.0, + max_abs: 1e9, +}; #[cfg(test)] mod tests; diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index 546ed0d4a..589619d3b 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -108,7 +108,7 @@ fn ddsketch_state_bytes_bound_the_library() { StateInputRows { rows_per_state, non_negative, - value_magnitude: None, + value_magnitude_range: None, }, ) .1 @@ -146,39 +146,39 @@ fn ddsketch_state_bytes_bound_the_library() { assert_eq!(priced(Some(10_000), false), priced(None, false)); } -/// A DDSketch over a column declaring its value magnitudes is priced at +/// A DDSketch over a column declaring its value magnitudes is costed at /// the bins they span, not nine decades. At α = 0.01, γ = 1.01 / 0.99 and /// a value `v` falls in bin `floor(ln v / ln γ)`: magnitudes in /// [0.001, 10000] take bins -346 to 460, 807 bins, where the default /// [1, 1e9] takes 0 to 1036, 1,037. With three or more non-negative rows /// the library's bound is `2 (bins + 126)` 8-byte slots plus the struct. #[test] -fn ddsketch_state_bytes_follow_the_declared_value_magnitude() { +fn ddsketch_state_bytes_follow_the_declared_value_magnitude_range() { let alpha = 0.01; - let priced = |value_magnitude| { + let costed = |value_magnitude_range| { sketch_shape( &SketchParams::DDSketch { alpha }, StateInputRows { rows_per_state: Some(600), non_negative: true, - value_magnitude, + value_magnitude_range, }, ) .1 }; let struct_bytes = std::mem::size_of::() as u64; let bytes_for_bins = |bins: u64| struct_bytes + 8 * 2 * (bins + 126); - let latency = ValueMagnitude { + let latency = ValueMagnitudeRange { min_abs: 0.001, max_abs: 10_000.0, }; - assert_eq!(priced(Some(latency)), bytes_for_bins(807)); - assert_eq!(priced(None), bytes_for_bins(1_037)); - // Values a narrower range holds fit the priced bytes. + assert_eq!(costed(Some(latency)), bytes_for_bins(807)); + assert_eq!(costed(None), bytes_for_bins(1_037)); + // Values a narrower range holds fit the costed bytes. let values: Vec = (0..600) .map(|i| 10f64.powf(-3.0 + 7.0 * f64::from(i) / 599.0)) .collect(); - assert!(library_ddsketch_bytes(alpha, &values) <= priced(Some(latency))); + assert!(library_ddsketch_bytes(alpha, &values) <= costed(Some(latency))); } /// A KLL state is charged what `asap_sketchlib` allocates for the @@ -204,7 +204,7 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { StateInputRows { rows_per_state, non_negative: true, - value_magnitude: None, + value_magnitude_range: None, }, ) .1 @@ -234,7 +234,7 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { /// Plan selection prices each state the executor builds per group at /// what the executor charges for it (#947). Each family is built /// through the executor's factory and fed `n` rows shaped as the -/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_VALUE_RANGE`], +/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_MAGNITUDE`], /// label values of [`ASSUMED_VALUE_KEY_BYTES`] keys for UnivMon, or /// keyed rows whose tuple keys are [`ASSUMED_HEAP_KEY_BYTES`] long. The /// heap families' shapes are dominated by their heaps. The charge is @@ -403,7 +403,7 @@ fn state_bytes_bound_the_executors_charge() { // scrambled order, alternating in sign. Rows::Values => { let at = f64::from(i * 7_919 % n) / f64::from(n); - let magnitude = DDSKETCH_ASSUMED_VALUE_RANGE.powf(at); + let magnitude = DDSKETCH_ASSUMED_MAGNITUDE.max_abs.powf(at); let sign = if i % 2 == 0 { 1.0 } else { -1.0 }; updater.update_single(sign * magnitude, 0).unwrap(); } @@ -421,7 +421,7 @@ fn state_bytes_bound_the_executors_charge() { StateInputRows { rows_per_state: Some(u64::from(n)), non_negative: false, - value_magnitude: None, + value_magnitude_range: None, }, ) .1 as usize; diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index bb2592668..2962811c0 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -44,7 +44,7 @@ use asap_types::plan::physical_asap_dag::{ use asap_types::stages::{Lifetime, StorageMedium}; use asap_types::summary_type::{SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate}; use asap_types::workload::bounds::{ - declared_non_negative, declared_series, declared_value_magnitude, group_count_bound, + declared_non_negative, declared_series, declared_value_magnitude_range, group_count_bound, source_name, source_rows_per_ms, unchanged_source_columns, }; use asap_types::workload::{DataWorkload, ONE_SECOND_MS}; @@ -482,35 +482,32 @@ pub fn node_resource_usage_and_cost<'a>( .ok_or(AnalyticalCostError::Overflow)?, }) }; - // Whether a summary built here sees raw data samples (its producer a - // scan or a time range) of sources all declared non-negative. - let raw_data_samples_non_negative = || { - let raw = producers.first().is_some_and(|producer| { - matches!( - producer.payload, - Payload::NonASAP(NonASAPOp::Scan { .. } | NonASAPOp::TimeRange { .. }) - ) - }); - raw && sources - .iter() - .all(|source| declared_non_negative(data, source)) - }; - // The declared magnitudes of the values a summary built here folds: - // its `weight` column, when the producer passes it on unchanged from - // the sources and every one of them declares its `column_magnitudes`. - // E.g. a DDSketch over a scan of `latency` declaring `value` in - // [0.001, 10000]. - let folded_value_magnitude = |update: &SummaryUpdate| { + // The source column whose values a summary built here folds: its + // `weight` column, when the producer passes that column on unchanged + // from the sources. E.g. `value` for a DDSketch over a scan of `latency` + // or over a filter on that scan; `None` for a computed weight such as a + // rate. + let folded_source_column = |update: &SummaryUpdate| -> Option<&str> { let SummaryInputExpr::Column(column) = &update.weight else { return None; }; let producer = producers.first()?; let index = resolve_column_ref(column, producer.output_schema).ok()?; let name = producer.output_schema.fields.get(index)?.name.as_str(); - if !producer.unchanged_columns.contains(name) { - return None; - } - declared_value_magnitude(data, sources.iter().copied(), name) + producer.unchanged_columns.contains(name).then_some(name) + }; + // What the sources declare about the values a summary built here folds: + // whether every one is non-negative, and their magnitude range, both + // from the one folded source column. + let folded_values = |update: &SummaryUpdate| { + folded_source_column(update).map_or((false, None), |column| { + let non_negative = !sources.is_empty() + && sources + .iter() + .all(|source| declared_non_negative(data, source, column)); + let range = declared_value_magnitude_range(data, sources.iter().copied(), column); + (non_negative, range) + }) }; let unary = |output| UnaryEdgeStatistics { input, @@ -833,10 +830,11 @@ pub fn node_resource_usage_and_cost<'a>( // At query time, each state sees its group's input rows; at // ingestion time, the rows of a window one second of // arriving rows does not bound. + let (non_negative, value_magnitude_range) = folded_values(update); let input_rows = StateInputRows { rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), - value_magnitude: folded_value_magnitude(update), + non_negative, + value_magnitude_range, }; let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); @@ -880,10 +878,11 @@ pub fn node_resource_usage_and_cost<'a>( .map_err(|_| AnalyticalCostError::Overflow)?, false => input.rows.div_ceil(retained_output_count), }; + let (non_negative, value_magnitude_range) = folded_values(update); let input_rows = StateInputRows { rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), - value_magnitude: folded_value_magnitude(update), + non_negative, + value_magnitude_range, }; let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 626091fa7..79b0322d6 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -487,97 +487,210 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { assert_eq!(usage.peak_memory_bytes, 16_000 * 66); } -/// A DDSketch built over a scan's `value` is priced at the bins its -/// source's declared `column_magnitudes` span: 807 at α = 0.01 for -/// [0.001, 10000], against 1,037 for the assumed [1, 1e9]. Undeclared, or -/// a column its producer does not pass on unchanged, keeps the assumed -/// range. -#[test] -fn a_ddsketch_over_a_scan_is_priced_at_its_declared_value_magnitude() { - use crate::node::state_bytes::{sketch_shape, StateInputRows}; +/// The bytes of a DDSketch state at α = 0.01 built over `column` of the +/// 600 rows `producer` emits from `source`, which passes on +/// `unchanged_columns` unchanged. +fn ddsketch_state_bytes( + data: &DataWorkload, + producer: Payload, + source: &str, + column: &str, + unchanged_columns: BTreeSet<&str>, +) -> u64 { use asap_types::ir::schema::{DataType, Field}; use asap_types::summary_type::SketchAlgorithm; - use asap_types::workload::{SourceFacts, ValueMagnitude}; - let settings = CostInputs::default(); - let alpha = 0.01; - let latency = ValueMagnitude { - min_abs: 0.001, - max_abs: 10_000.0, - }; - let data = |magnitude: Option| DataWorkload { - sources: BTreeMap::from([( - "latency".to_string(), - SourceFacts { - column_magnitudes: magnitude - .map(|magnitude| ("value".to_string(), magnitude)) - .into_iter() - .collect(), - ..SourceFacts::default() - }, - )]), - ..DataWorkload::default() - }; - let scan = PhysicalASAPDAGNode { + let producer = PhysicalASAPDAGNode { output_schema: Schema::new(vec![ Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), + Field::plain(column, DataType::Float64, false), ]), - ..node(Payload::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: vec![], - schema: Schema::default(), - })) + ..node(producer) }; let build = PhysicalASAPDAGNode { - parameters: Some(SketchParams::DDSketch { alpha }), + parameters: Some(SketchParams::DDSketch { alpha: 0.01 }), ..node(Payload::ASAP(ASAPOp::SummaryAgg { child: 0, summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), - input: SummaryUpdate::column(ColumnRef::SampleValue), + input: SummaryUpdate::column(ColumnRef::Named(column.to_string())), reduction: Reduction::Reduce(GroupKeys::default()), filter: None, })) }; - let state_bytes = |data: &DataWorkload, unchanged_columns: BTreeSet<&str>| { - let scan_output = ProducerOutput { - payload: &scan.payload, - output_schema: &scan.output_schema, - output: EdgeStatistics { - rows: 600, - bytes: 9_600, - }, - sources: BTreeSet::from(["latency"]), - family: StateFamily::None, - unchanged_columns, - tumbling_windows: 1, - }; - node_resource_usage_and_cost(&build, &[scan_output], (1, 1.0), None, data, &settings) - .unwrap() - .output - .bytes + let producer_output = ProducerOutput { + payload: &producer.payload, + output_schema: &producer.output_schema, + output: EdgeStatistics { + rows: 600, + bytes: 9_600, + }, + sources: BTreeSet::from([source]), + family: StateFamily::None, + unchanged_columns, + tumbling_windows: 1, }; - let priced = |magnitude| { - sketch_shape( - &SketchParams::DDSketch { alpha }, - StateInputRows { - rows_per_state: Some(600), - non_negative: false, - value_magnitude: magnitude, + let settings = CostInputs::default(); + node_resource_usage_and_cost(&build, &[producer_output], (1, 1.0), None, data, &settings) + .unwrap() + .output + .bytes +} + +/// The bytes `sketch_shape` gives a DDSketch state at α = 0.01 over 600 +/// rows of the given sign and magnitude range. +fn ddsketch_costed_bytes( + non_negative: bool, + value_magnitude_range: Option, +) -> u64 { + use crate::node::state_bytes::{sketch_shape, StateInputRows}; + sketch_shape( + &SketchParams::DDSketch { alpha: 0.01 }, + StateInputRows { + rows_per_state: Some(600), + non_negative, + value_magnitude_range, + }, + ) + .1 +} + +/// A scan of PromQL metric `metric`. +fn metric_scan(metric: &str) -> Payload { + Payload::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::default(), + }) +} + +/// A data workload whose one source `source` declares `facts`. +fn declaring(source: &str, facts: asap_types::workload::SourceFacts) -> DataWorkload { + DataWorkload { + sources: BTreeMap::from([(source.to_string(), facts)]), + ..DataWorkload::default() + } +} + +/// A DDSketch built over a scan's `value` is costed at the bins its +/// source's declared `column_magnitudes` span: 807 at α = 0.01 for +/// [0.001, 10000], against 1,037 for the assumed [1, 1e9]. Undeclared, or +/// a column its producer does not pass on unchanged, keeps the assumed +/// range. +#[test] +fn a_ddsketch_over_a_scan_is_costed_at_its_declared_value_magnitude_range() { + use asap_types::workload::{SourceFacts, ValueMagnitudeRange}; + let latency = ValueMagnitudeRange { + min_abs: 0.001, + max_abs: 10_000.0, + }; + let data = |range: Option| { + declaring( + "latency", + SourceFacts { + column_magnitudes: range + .map(|range| ("value".to_string(), range)) + .into_iter() + .collect(), + ..SourceFacts::default() }, ) - .1 + }; + let state_bytes = |data: &DataWorkload, unchanged_columns| { + ddsketch_state_bytes( + data, + metric_scan("latency"), + "latency", + "value", + unchanged_columns, + ) }; let unchanged = || BTreeSet::from(["ts", "value"]); assert_eq!( state_bytes(&data(Some(latency)), unchanged()), - priced(Some(latency)) + ddsketch_costed_bytes(false, Some(latency)) + ); + assert_eq!( + state_bytes(&data(None), unchanged()), + ddsketch_costed_bytes(false, None) ); - assert_eq!(state_bytes(&data(None), unchanged()), priced(None)); assert_eq!( state_bytes(&data(Some(latency)), BTreeSet::from(["ts"])), - priced(None) + ddsketch_costed_bytes(false, None) + ); + assert!(ddsketch_costed_bytes(false, Some(latency)) < ddsketch_costed_bytes(false, None)); +} + +/// A DDSketch over a SQL table's `latency` column that the table declares +/// non-negative keeps one store: the sign is read for the folded column, +/// not for a PromQL `value`. Undeclared, it keeps both stores. +#[test] +fn a_ddsketch_over_a_sql_column_declared_non_negative_keeps_one_store() { + use asap_types::stages::ValueSign; + use asap_types::workload::SourceFacts; + let table_scan = || { + Payload::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "requests".into(), + }, + predicates: vec![], + schema: Schema::default(), + }) + }; + let declared = declaring( + "requests", + SourceFacts { + column_signs: BTreeMap::from([("latency".to_string(), ValueSign::NonNegative)]), + ..SourceFacts::default() + }, + ); + let state_bytes = |data: &DataWorkload| { + ddsketch_state_bytes( + data, + table_scan(), + "requests", + "latency", + BTreeSet::from(["ts", "latency"]), + ) + }; + assert_eq!(state_bytes(&declared), ddsketch_costed_bytes(true, None)); + assert_eq!( + state_bytes(&DataWorkload::default()), + ddsketch_costed_bytes(false, None) + ); + assert!(ddsketch_costed_bytes(true, None) < ddsketch_costed_bytes(false, None)); +} + +/// A DDSketch over a filter on a scan of a source declared non-negative +/// keeps one store: the filter passes `value` on unchanged. A producer +/// that computes `value` (not among its unchanged columns) keeps both. +#[test] +fn a_ddsketch_over_a_filter_on_a_non_negative_scan_keeps_one_store() { + use asap_types::ir::scalar::{Predicate, ScalarExpr, ScalarValue}; + use asap_types::stages::ValueSign; + use asap_types::workload::SourceFacts; + let filter = || { + Payload::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + child: 0, + }) + }; + let data = declaring( + "latency", + SourceFacts { + column_signs: BTreeMap::from([("value".to_string(), ValueSign::NonNegative)]), + ..SourceFacts::default() + }, + ); + let state_bytes = |unchanged_columns| { + ddsketch_state_bytes(&data, filter(), "latency", "value", unchanged_columns) + }; + assert_eq!( + state_bytes(BTreeSet::from(["ts", "value"])), + ddsketch_costed_bytes(true, None) + ); + assert_eq!( + state_bytes(BTreeSet::from(["ts"])), + ddsketch_costed_bytes(false, None) ); - assert!(priced(Some(latency)) < priced(None)); } diff --git a/crates/frontend-promql/tests/histogram_metadata.rs b/crates/frontend-promql/tests/histogram_metadata.rs index 1fa25858f..8b44fbcea 100644 --- a/crates/frontend-promql/tests/histogram_metadata.rs +++ b/crates/frontend-promql/tests/histogram_metadata.rs @@ -183,9 +183,9 @@ fn declared_summary_family() { Summary )); let data = over_family("", "foo", Summary).data_workload.unwrap(); - assert!(declared_non_negative(&data, "foo_sum")); - assert!(declared_non_negative(&data, "foo_count")); - assert!(!declared_non_negative(&data, "foo")); + assert!(declared_non_negative(&data, "foo_sum", "value")); + assert!(declared_non_negative(&data, "foo_count", "value")); + assert!(!declared_non_negative(&data, "foo", "value")); } /// A declared histogram family `foo`: `foo_sum` and `foo_count` read as @@ -203,6 +203,6 @@ fn declared_histogram_family() { Histogram )); let data = over_family("", "foo", Histogram).data_workload.unwrap(); - assert!(declared_non_negative(&data, "foo_sum")); - assert!(declared_non_negative(&data, "foo_count")); + assert!(declared_non_negative(&data, "foo_sum", "value")); + assert!(declared_non_negative(&data, "foo_count", "value")); } diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index eebdbe0af..2d2136083 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -12,7 +12,7 @@ use crate::ir::{ }; use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; use crate::summary_type::SketchStatistic; -use crate::workload::{DataWorkload, PrometheusMetricType, ValueMagnitude}; +use crate::workload::{DataWorkload, PrometheusMetricType, ValueMagnitudeRange}; /// A scan source's name: its PromQL metric or SQL table. pub fn source_name(source: &Source) -> &str { @@ -405,40 +405,49 @@ fn at_most_one_row(node: &OperatorNode) -> bool { } } -/// Whether `data` declares every sample of `metric` non-negative: the -/// metric is declared a counter, or its `value` column is declared -/// non-negative ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). -pub fn declared_non_negative(data: &DataWorkload, metric: &str) -> bool { - declared_counter(data, metric) - || data.series_facts(metric).is_some_and(|(facts, _)| { - facts.column_signs.get("value") == Some(&crate::stages::ValueSign::NonNegative) +/// Whether `data` declares every value of `source`'s `column` non-negative: +/// the column is declared non-negative +/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), +/// or it is a PromQL sample's `value` and the source is declared a counter. +/// A histogram or summary family's signs apply to its series +/// ([`DataWorkload::series_facts`]). E.g. `latency` declaring +/// `{"value": "non_negative"}` gives true for `value`, and a SQL table +/// declaring `{"latency": "non_negative"}` gives true for `latency`. +pub fn declared_non_negative(data: &DataWorkload, source: &str, column: &str) -> bool { + (column == "value" && declared_counter(data, source)) + || data.series_facts(source).is_some_and(|(facts, _)| { + facts.column_signs.get(column) == Some(&crate::stages::ValueSign::NonNegative) }) } -/// The magnitudes of `column`'s non-zero values over every source in -/// `sources`, from each one's declared `column_magnitudes` +/// The range of magnitudes of `column`'s non-zero values over every source +/// in `sources`, from each one's own declared `column_magnitudes` /// ([`SourceFacts::column_magnitudes`](crate::workload::SourceFacts::column_magnitudes)): -/// the smallest `min_abs` and the largest `max_abs` among them. `None` when -/// `sources` is empty or some source declares none for `column`. E.g. -/// sources declaring `[0.001, 10]` and `[0.01, 100]` give `[0.001, 100]`. -pub fn declared_value_magnitude<'a>( +/// the smallest `min_abs` and the largest `max_abs` among them. Each series +/// is read under its own name, so a histogram family `foo`'s range does not +/// bound `foo_sum`, `foo_count` or `foo_bucket`. `None` when `sources` is +/// empty or some source declares none for `column`. E.g. sources declaring +/// `[0.001, 10]` and `[0.01, 100]` give `[0.001, 100]`. +pub fn declared_value_magnitude_range<'a>( data: &DataWorkload, sources: impl IntoIterator, column: &str, -) -> Option { +) -> Option { sources .into_iter() - .try_fold(None, |union: Option, source| { - let (facts, _) = data.series_facts(source)?; - let declared = *facts.column_magnitudes.get(column)?; - Some(Some(match union { - None => declared, - Some(union) => ValueMagnitude { - min_abs: union.min_abs.min(declared.min_abs), - max_abs: union.max_abs.max(declared.max_abs), - }, - })) - })? + .map(|source| { + data.sources + .get(source)? + .column_magnitudes + .get(column) + .copied() + }) + .collect::>>()? + .into_iter() + .reduce(|union, declared| ValueMagnitudeRange { + min_abs: union.min_abs.min(declared.min_abs), + max_abs: union.max_abs.max(declared.max_abs), + }) } /// Whether `data` declares `metric` a counter, itself or as the `_sum` or @@ -712,7 +721,7 @@ mod tests { let facts = |min_abs, max_abs| SourceFacts { column_magnitudes: BTreeMap::from([( "value".to_string(), - ValueMagnitude { min_abs, max_abs }, + ValueMagnitudeRange { min_abs, max_abs }, )]), ..SourceFacts::default() }; @@ -725,11 +734,11 @@ mod tests { ..DataWorkload::default() }; let magnitude = |sources: &[&str], column| { - declared_value_magnitude(&data, sources.iter().copied(), column) + declared_value_magnitude_range(&data, sources.iter().copied(), column) }; assert_eq!( magnitude(&["a", "b"], "value"), - Some(ValueMagnitude { + Some(ValueMagnitudeRange { min_abs: 0.001, max_abs: 100.0 }) @@ -739,6 +748,50 @@ mod tests { assert_eq!(magnitude(&[], "value"), None); } + /// A histogram family's declared range bounds its observations, not + /// the counts in `foo_count` or the sums in `foo_sum`: each series' + /// range is read from its own entry. Here `foo` declares observations + /// in [0.001, 10] and `foo_count` its counts in [1, 1e6]. + #[test] + fn a_series_magnitudes_come_from_its_own_entry_not_its_family() { + let magnitudes = |min_abs, max_abs| { + BTreeMap::from([( + "value".to_string(), + ValueMagnitudeRange { min_abs, max_abs }, + )]) + }; + let data = DataWorkload { + sources: BTreeMap::from([ + ( + "foo".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Histogram), + column_magnitudes: magnitudes(0.001, 10.0), + ..SourceFacts::default() + }, + ), + ( + "foo_count".to_string(), + SourceFacts { + column_magnitudes: magnitudes(1.0, 1e6), + ..SourceFacts::default() + }, + ), + ]), + ..DataWorkload::default() + }; + let magnitude = |source| declared_value_magnitude_range(&data, [source], "value"); + assert_eq!( + magnitude("foo_count"), + Some(ValueMagnitudeRange { + min_abs: 1.0, + max_abs: 1e6 + }) + ); + assert_eq!(magnitude("foo_sum"), None); + assert_eq!(magnitude("foo_bucket"), None); + } + /// A source's rows per ms: its series over its sample interval when it /// declares both, else its series share of the workload's rate. #[test] diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index cb7d3356f..7ce5f9a78 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -91,7 +91,7 @@ pub struct DataWorkload { /// family name; its `metric_type`, `column_signs` and /// `sample_interval` apply to every series of the family /// ([`Self::series_facts`]). Counts, norms and bounds (`series_count`, - /// `stream_norms`, …) differ per series name (a `foo_bucket` has one + /// `stream_norms`, `column_magnitudes`, …) differ per series name (a `foo_bucket` has one /// series per `le`), so they are read from the series' own entry; /// pricing a source's own arrival rate reads its `series_count` and /// `sample_interval` there together. A source without an entry has no @@ -128,13 +128,17 @@ pub struct SourceFacts { pub column_signs: BTreeMap, /// Bounds on the magnitude of each column's non-zero values, keyed by /// column name (default: none declared), e.g. `{"value": {"min_abs": - /// 0.001, "max_abs": 10000.0}}` for a latency in seconds. A DDSketch - /// state built from the column unchanged is priced at the - /// `log_γ(max_abs / min_abs)` bins those magnitudes span instead of an - /// assumed range of nine decades. The deployment must ensure it. - /// Omitted from JSON when empty. + /// 0.001, "max_abs": 10000.0}}` for a latency in seconds. Read per + /// series name: a histogram or summary family's range does not apply to + /// its `_sum`, `_count` or `_bucket` series. A DDSketch state built from + /// the column unchanged is costed at the bins those magnitudes span (the + /// bucket indices from ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one) + /// instead of an assumed range of nine decades. The deployment must + /// ensure it: a range that is too narrow can make Stage 3 pick a finer + /// and much larger sketch than the default range would. Omitted from + /// JSON when empty. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub column_magnitudes: BTreeMap, + pub column_magnitudes: BTreeMap, /// How often each series of the metric (or group of rows of a SQL /// table) receives a sample (default: unknown). A PromQL or MetricsQL /// bare instant selector over the metric reads the latest sample within it; lowering such a @@ -211,7 +215,7 @@ pub struct SourceFacts { /// "max_abs": 10000.0}` for latencies from 1 ms to about 3 h in seconds. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ValueMagnitude { +pub struct ValueMagnitudeRange { /// The smallest magnitude of a non-zero value. pub min_abs: f64, /// The largest magnitude of a value. @@ -395,13 +399,13 @@ impl DataWorkload { } } for (column, magnitude) in &facts.column_magnitudes { - let ValueMagnitude { min_abs, max_abs } = *magnitude; + let ValueMagnitudeRange { min_abs, max_abs } = *magnitude; if !(min_abs.is_finite() && max_abs.is_finite() && 0.0 < min_abs && min_abs <= max_abs) { - return Err(WorkloadError::InvalidValueMagnitude { + return Err(WorkloadError::InvalidValueMagnitudeRange { source_name: source_name.clone(), column: column.clone(), min_abs, @@ -554,7 +558,7 @@ mod tests { SourceFacts { column_magnitudes: BTreeMap::from([( "value".to_string(), - ValueMagnitude { min_abs, max_abs }, + ValueMagnitudeRange { min_abs, max_abs }, )]), ..SourceFacts::default() }, @@ -575,7 +579,7 @@ mod tests { assert!( matches!( &error, - WorkloadError::InvalidValueMagnitude { source_name, column, .. } + WorkloadError::InvalidValueMagnitudeRange { source_name, column, .. } if source_name == "latency" && column == "value" ), "{min_abs}, {max_abs}: {error:?}" diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index ab90b157c..96dd88390 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -11,7 +11,7 @@ mod query; pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, - StreamNorms, ValueMagnitude, ONE_SECOND_MS, + StreamNorms, ValueMagnitudeRange, ONE_SECOND_MS, }; pub use planning::{PlanningWorkload, WorkloadError}; pub use query::{ diff --git a/crates/types/src/workload/planning.rs b/crates/types/src/workload/planning.rs index 290031af9..cf735ce22 100644 --- a/crates/types/src/workload/planning.rs +++ b/crates/types/src/workload/planning.rs @@ -140,7 +140,7 @@ pub enum WorkloadError { "source {source_name}: column_magnitudes for column {column} must have \ 0 < min_abs <= max_abs, both finite, got min_abs {min_abs}, max_abs {max_abs}" )] - InvalidValueMagnitude { + InvalidValueMagnitudeRange { source_name: String, column: String, min_abs: f64, @@ -201,7 +201,7 @@ mod tests { )]), column_magnitudes: BTreeMap::from([( "value".to_string(), - ValueMagnitude { + ValueMagnitudeRange { min_abs: 1.0, max_abs: 1e12, }, diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index dbbd43313..75e2c9bdb 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -197,7 +197,7 @@ struct SourceFacts { series_count: Option, metric_type: Option, column_signs: BTreeMap, - column_magnitudes: BTreeMap, + column_magnitudes: BTreeMap, sample_interval: Option, stream_norms: Option, max_distinct_per_evaluation: Option, @@ -205,7 +205,7 @@ struct SourceFacts { distinct_values_per_label_set: Vec, rows_held: Option, } -struct ValueMagnitude { +struct ValueMagnitudeRange { min_abs: f64, max_abs: f64, } @@ -220,9 +220,9 @@ struct LabelSetDistinctValues { | `arrival` | Present; may be `Unknown` | Distinguishes data at rest, ingesting data, and mixed data. Ingestion-time builds are possible only for ingesting or mixed data. | | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | -| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | +| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms, bounds and `column_magnitudes` are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | | `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | -| `column_magnitudes` | Optional; default none; each `0 < min_abs <= max_abs`, both finite | Bounds on the magnitude of each column's non-zero values, keyed by column name, e.g. `{"value": {"min_abs": 0.001, "max_abs": 10000.0}}` for a latency in seconds. A DDSketch state built from a column its producer passes on unchanged from sources that all declare it is priced at the `log_γ(max_abs / min_abs)` bins that range spans (over several sources, the widest range among them), instead of an assumed range of nine decades. The deployment must ensure it. Any other value is an input error naming the source and column. | +| `column_magnitudes` | Optional; default none; each `0 < min_abs <= max_abs`, both finite | Bounds on the magnitude of each column's non-zero values, keyed by column name, e.g. `{"value": {"min_abs": 0.001, "max_abs": 10000.0}}` for a latency in seconds. Read per series name: a `histogram` or `summary` family's range does not apply to its `_sum`, `_count` or `_bucket` series. A DDSketch state built from a column its producer passes on unchanged from sources that all declare it is costed at the bins that range spans (over several sources, the widest range among them): the bucket indices from ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one, for γ = (1 + α) / (1 − α), instead of an assumed range of nine decades. The same column's declared `column_signs` decides whether the sketch keeps a negative store. The deployment must ensure the range: one that is too narrow can make Stage 3 pick a finer and much larger sketch than the default range would. Any other value is an input error naming the source and column. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 658b3c214..1932537c6 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -308,8 +308,8 @@ query's window comes from its text (range and offset). These facts are separate: PromQL metric or SQL table name to `SourceFacts`: `series_count`, `metric_type`, `column_signs`, `column_magnitudes` (bounds on each column's non-zero value magnitudes, `{"value": {"min_abs": 0.001, -"max_abs": 10000.0}}`, which price a DDSketch state at the bins that range -spans instead of nine decades), `sample_interval` (PromQL and +"max_abs": 10000.0}}`, read per series name, which cost a DDSketch state +at the bins that range spans instead of nine decades), `sample_interval` (PromQL and MetricsQL), `stream_norms`, `max_distinct_per_evaluation` and `distinct_values_per_label` (distinct values per label or column, which plan selection prices a `by (...)` grouping's groups at: the product of its From 59dc20d7f9a7e1da8164e41b92bf6d7877493104 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:29:59 +0000 Subject: [PATCH 4/6] fix(types, cost): one declared-sign rule for the memo's Scan and DDSketch cost - Add workload::bounds::declared_sign (and declared_column_sign for the memo, which holds only the sources): the column_signs entry, else NonNegative for a counter's value, else Unknown. The memo's Scan arm and declared_non_negative both use it, so a counter declaring {"value": "unknown"} is Unknown in the memo and its DDSketch is costed with two stores. declared_counter is deleted. - Move the folded-column rule to ProducerOutput::unchanged_column. - Fold declared_value_magnitude_range without collecting a Vec. - Rename DDSKETCH_ASSUMED_MAGNITUDE to DDSKETCH_ASSUMED_MAGNITUDE_RANGE and the union test to declared_value_magnitude_ranges_... - Docs: column_signs is read by DDSketch costing too. --- crates/cost/src/node/state_bytes/mod.rs | 14 +-- crates/cost/src/node/state_bytes/tests.rs | 4 +- crates/cost/src/node/usage/mod.rs | 52 ++++++----- crates/cost/src/node/usage/tests.rs | 40 +++++++++ .../stages/logical/group/column_properties.rs | 21 ++--- crates/types/src/workload/bounds.rs | 89 +++++++++++-------- crates/types/src/workload/data.rs | 20 +++-- crates/types/tests/memo_group_properties.rs | 40 +++++++++ .../architecture/input-output-workflow.md | 2 +- 9 files changed, 188 insertions(+), 94 deletions(-) diff --git a/crates/cost/src/node/state_bytes/mod.rs b/crates/cost/src/node/state_bytes/mod.rs index 955fe3104..fac0cc11e 100644 --- a/crates/cost/src/node/state_bytes/mod.rs +++ b/crates/cost/src/node/state_bytes/mod.rs @@ -45,8 +45,8 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// - DDSketch and KMV: the rows the state folds, at least one (any number /// when unknown), each a distinct item for KMV; a DDSketch's values span /// `input_rows.value_magnitude_range` when declared, else -/// `DDSKETCH_ASSUMED_MAGNITUDE`, and fill its negative store too unless -/// proven non-negative. Its bins are the bucket indices from +/// `DDSKETCH_ASSUMED_MAGNITUDE_RANGE`, and fill its negative store too +/// unless proven non-negative. Its bins are the bucket indices from /// ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one, for γ = (1 + α) / /// (1 - α); this DDSketch has no bin limit to cap them. E.g. α = 0.01 /// over [0.001, 10000] takes indices -346 to 460, 807 bins. @@ -97,7 +97,7 @@ pub struct StateInputRows { pub non_negative: bool, /// Bounds on `|value|` of the non-zero values folded into the state; /// `None` when not declared, which costs a DDSketch over the assumed - /// `DDSKETCH_ASSUMED_MAGNITUDE`. E.g. `Some(ValueMagnitudeRange { + /// `DDSKETCH_ASSUMED_MAGNITUDE_RANGE`. E.g. `Some(ValueMagnitudeRange { /// min_abs: 0.001, max_abs: 10000.0 })` for a latency in seconds. pub value_magnitude_range: Option, } @@ -175,7 +175,7 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, SketchParams::DDSketch { alpha } => { let ValueMagnitudeRange { min_abs, max_abs } = input_rows .value_magnitude_range - .unwrap_or(DDSKETCH_ASSUMED_MAGNITUDE); + .unwrap_or(DDSKETCH_ASSUMED_MAGNITUDE_RANGE); ( 1.0, DdSketch::estimated_memory_bytes( @@ -287,11 +287,11 @@ fn univmon_expected_layers(layers: u8) -> f64 { 2.0 - 2f64.powi(1 - i32::from(layers.max(1))) } -/// The value magnitudes a DDSketch is assumed to see when its column -/// declares no `column_magnitudes`: nine decades, e.g. latencies from 1 ns +/// The range of value magnitudes a DDSketch is assumed to see when its +/// column declares no `column_magnitudes`: nine decades, e.g. latencies from 1 ns /// to 1 s, costed as magnitudes in `[1, 1e9]` (its bins depend on the ratio /// alone). An assumption, not workload evidence. -const DDSKETCH_ASSUMED_MAGNITUDE: ValueMagnitudeRange = ValueMagnitudeRange { +const DDSKETCH_ASSUMED_MAGNITUDE_RANGE: ValueMagnitudeRange = ValueMagnitudeRange { min_abs: 1.0, max_abs: 1e9, }; diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index 589619d3b..852f6fe52 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -234,7 +234,7 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { /// Plan selection prices each state the executor builds per group at /// what the executor charges for it (#947). Each family is built /// through the executor's factory and fed `n` rows shaped as the -/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_MAGNITUDE`], +/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_MAGNITUDE_RANGE`], /// label values of [`ASSUMED_VALUE_KEY_BYTES`] keys for UnivMon, or /// keyed rows whose tuple keys are [`ASSUMED_HEAP_KEY_BYTES`] long. The /// heap families' shapes are dominated by their heaps. The charge is @@ -403,7 +403,7 @@ fn state_bytes_bound_the_executors_charge() { // scrambled order, alternating in sign. Rows::Values => { let at = f64::from(i * 7_919 % n) / f64::from(n); - let magnitude = DDSKETCH_ASSUMED_MAGNITUDE.max_abs.powf(at); + let magnitude = DDSKETCH_ASSUMED_MAGNITUDE_RANGE.max_abs.powf(at); let sign = if i % 2 == 0 { 1.0 } else { -1.0 }; updater.update_single(sign * magnitude, 0).unwrap(); } diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 2962811c0..efc2bd383 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -183,6 +183,22 @@ pub struct ProducerOutput<'a> { pub chain_tumbling_window_width_ms: Option, } +impl ProducerOutput<'_> { + /// The source column `weight` reads, when this producer passes that + /// column on unchanged from the sources: the column whose values a + /// summary built over this producer folds. E.g. `value` for a DDSketch + /// over a scan of `latency` or over a filter on that scan; `None` for a + /// computed weight such as a rate. + pub(crate) fn unchanged_column(&self, weight: &SummaryInputExpr) -> Option<&str> { + let SummaryInputExpr::Column(column) = weight else { + return None; + }; + let index = resolve_column_ref(column, self.output_schema).ok()?; + let name = self.output_schema.fields.get(index)?.name.as_str(); + self.unchanged_columns.contains(name).then_some(name) + } +} + /// What one node uses and outputs ([`node_resource_usage_and_cost`]): its /// output statistics, sources and state family for its consumers; its work /// for one computation (one execution) in ms on one core; its usage per @@ -482,32 +498,22 @@ pub fn node_resource_usage_and_cost<'a>( .ok_or(AnalyticalCostError::Overflow)?, }) }; - // The source column whose values a summary built here folds: its - // `weight` column, when the producer passes that column on unchanged - // from the sources. E.g. `value` for a DDSketch over a scan of `latency` - // or over a filter on that scan; `None` for a computed weight such as a - // rate. - let folded_source_column = |update: &SummaryUpdate| -> Option<&str> { - let SummaryInputExpr::Column(column) = &update.weight else { - return None; - }; - let producer = producers.first()?; - let index = resolve_column_ref(column, producer.output_schema).ok()?; - let name = producer.output_schema.fields.get(index)?.name.as_str(); - producer.unchanged_columns.contains(name).then_some(name) - }; // What the sources declare about the values a summary built here folds: // whether every one is non-negative, and their magnitude range, both - // from the one folded source column. + // from the one source column it folds + // (`ProducerOutput::unchanged_column`). let folded_values = |update: &SummaryUpdate| { - folded_source_column(update).map_or((false, None), |column| { - let non_negative = !sources.is_empty() - && sources - .iter() - .all(|source| declared_non_negative(data, source, column)); - let range = declared_value_magnitude_range(data, sources.iter().copied(), column); - (non_negative, range) - }) + producers + .first() + .and_then(|producer| producer.unchanged_column(&update.weight)) + .map_or((false, None), |column| { + let non_negative = !sources.is_empty() + && sources + .iter() + .all(|source| declared_non_negative(data, source, column)); + let range = declared_value_magnitude_range(data, sources.iter().copied(), column); + (non_negative, range) + }) }; let unary = |output| UnaryEdgeStatistics { input, diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 79b0322d6..781491e2c 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -694,3 +694,43 @@ fn a_ddsketch_over_a_filter_on_a_non_negative_scan_keeps_one_store() { ddsketch_costed_bytes(false, None) ); } + +/// A counter's `value` is non-negative unless the source declares another +/// sign for it: a counter `requests_total` declaring `{"value": "unknown"}` +/// keeps both stores, the sign the memo gives its `Scan` too. Without the +/// declaration it keeps one store. +#[test] +fn a_ddsketch_over_a_counter_declaring_its_value_unknown_keeps_both_stores() { + use asap_types::stages::ValueSign; + use asap_types::workload::{PrometheusMetricType, SourceFacts}; + let counter = |column_signs| { + declaring( + "requests_total", + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + column_signs, + ..SourceFacts::default() + }, + ) + }; + let state_bytes = |data: &DataWorkload| { + ddsketch_state_bytes( + data, + metric_scan("requests_total"), + "requests_total", + "value", + BTreeSet::from(["ts", "value"]), + ) + }; + assert_eq!( + state_bytes(&counter(BTreeMap::from([( + "value".to_string(), + ValueSign::Unknown + )]))), + ddsketch_costed_bytes(false, None) + ); + assert_eq!( + state_bytes(&counter(BTreeMap::new())), + ddsketch_costed_bytes(true, None) + ); +} diff --git a/crates/types/src/stages/logical/group/column_properties.rs b/crates/types/src/stages/logical/group/column_properties.rs index 8bab8f34d..b3dd1dd6d 100644 --- a/crates/types/src/stages/logical/group/column_properties.rs +++ b/crates/types/src/stages/logical/group/column_properties.rs @@ -26,8 +26,8 @@ use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, }; -use crate::workload::bounds::source_name; -use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; +use crate::workload::bounds::{declared_column_sign, source_name}; +use crate::workload::{series_facts, SourceFacts}; /// The sign of one column's values, a group property derived from the /// input groups like lineage (concepts doc, section 3, "Value sign"); a @@ -201,26 +201,15 @@ fn non_asap_columns( use NonASAPOp::*; match op { // Each column copied from its source, with the sign its source's - // facts declare (a histogram's or summary's series reads its - // family's); a counter's `value` is non-negative without a - // declaration. + // facts declare (`declared_sign`): e.g. a counter's `value` is + // non-negative unless it declares another sign. Scan { source, schema, .. } => { let facts = series_facts(source_facts, source_name(source)); schema .fields .iter() .map(|field| ColumnProperties { - sign: match facts { - Some((facts, series_type)) => facts - .column_signs - .get(&field.name) - .copied() - .or((field.name == "value" - && series_type == Some(PrometheusMetricType::Counter)) - .then_some(NonNegative)) - .unwrap_or(Unknown), - None => Unknown, - }, + sign: declared_column_sign(facts, &field.name), lineage: Some(ColumnLineage::SourceColumn(SourceColumn { source: source.clone(), scan_ordinal: 0, diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 2d2136083..32bff6d3b 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -11,8 +11,9 @@ use crate::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, ScalarExpr, }; use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; +use crate::stages::ValueSign; use crate::summary_type::SketchStatistic; -use crate::workload::{DataWorkload, PrometheusMetricType, ValueMagnitudeRange}; +use crate::workload::{DataWorkload, PrometheusMetricType, SourceFacts, ValueMagnitudeRange}; /// A scan source's name: its PromQL metric or SQL table. pub fn source_name(source: &Source) -> &str { @@ -405,19 +406,41 @@ fn at_most_one_row(node: &OperatorNode) -> bool { } } -/// Whether `data` declares every value of `source`'s `column` non-negative: -/// the column is declared non-negative -/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), -/// or it is a PromQL sample's `value` and the source is declared a counter. -/// A histogram or summary family's signs apply to its series -/// ([`DataWorkload::series_facts`]). E.g. `latency` declaring -/// `{"value": "non_negative"}` gives true for `value`, and a SQL table -/// declaring `{"latency": "non_negative"}` gives true for `latency`. +/// The sign `data` declares for `source`'s `column`: its +/// [`SourceFacts::column_signs`] entry (a histogram or summary family's +/// applies to its series, [`DataWorkload::series_facts`]), else +/// `NonNegative` for a counter's `value`, else `Unknown`. The memo signs a +/// `Scan`'s columns by this rule, and DDSketch costing reads it. E.g. a +/// counter `requests_total` gives `NonNegative` for `value`, or `Unknown` +/// when it declares `{"value": "unknown"}`; a SQL table declaring +/// `{"latency": "non_negative"}` gives `NonNegative` for `latency`. +pub fn declared_sign(data: &DataWorkload, source: &str, column: &str) -> ValueSign { + declared_column_sign(data.series_facts(source), column) +} + +/// [`declared_sign`] from the facts and series type +/// [`DataWorkload::series_facts`] finds for the source, for a caller that +/// holds only the sources: `Unknown` when there are none. +pub(crate) fn declared_column_sign( + series_facts: Option<(&SourceFacts, Option)>, + column: &str, +) -> ValueSign { + let Some((facts, series_type)) = series_facts else { + return ValueSign::Unknown; + }; + match facts.column_signs.get(column) { + Some(&declared) => declared, + None if column == "value" && series_type == Some(PrometheusMetricType::Counter) => { + ValueSign::NonNegative + } + None => ValueSign::Unknown, + } +} + +/// Whether `data` declares every value of `source`'s `column` non-negative +/// ([`declared_sign`]). pub fn declared_non_negative(data: &DataWorkload, source: &str, column: &str) -> bool { - (column == "value" && declared_counter(data, source)) - || data.series_facts(source).is_some_and(|(facts, _)| { - facts.column_signs.get(column) == Some(&crate::stages::ValueSign::NonNegative) - }) + declared_sign(data, source, column) == ValueSign::NonNegative } /// The range of magnitudes of `column`'s non-zero values over every source @@ -433,30 +456,22 @@ pub fn declared_value_magnitude_range<'a>( sources: impl IntoIterator, column: &str, ) -> Option { - sources - .into_iter() - .map(|source| { - data.sources - .get(source)? - .column_magnitudes - .get(column) - .copied() - }) - .collect::>>()? - .into_iter() - .reduce(|union, declared| ValueMagnitudeRange { - min_abs: union.min_abs.min(declared.min_abs), - max_abs: union.max_abs.max(declared.max_abs), + let declared = |source: &str| { + data.sources + .get(source)? + .column_magnitudes + .get(column) + .copied() + }; + let mut sources = sources.into_iter(); + let first = declared(sources.next()?)?; + sources.try_fold(first, |union, source| { + let range = declared(source)?; + Some(ValueMagnitudeRange { + min_abs: union.min_abs.min(range.min_abs), + max_abs: union.max_abs.max(range.max_abs), }) -} - -/// Whether `data` declares `metric` a counter, itself or as the `_sum` or -/// `_count` series of a declared histogram or summary -/// ([`DataWorkload::series_facts`]). A declared type does not expire, so its -/// evidence is not checked. -fn declared_counter(data: &DataWorkload, metric: &str) -> bool { - data.series_facts(metric) - .is_some_and(|(_, series_type)| series_type == Some(PrometheusMetricType::Counter)) + }) } /// Rows per millisecond the sources in `sources` receive together, each @@ -717,7 +732,7 @@ mod tests { /// The magnitudes over several sources span all of theirs, and are /// unknown when any of them declares none for the column. #[test] - fn declared_value_magnitudes_are_the_union_over_the_sources() { + fn declared_value_magnitude_ranges_are_the_union_over_the_sources() { let facts = |min_abs, max_abs| SourceFacts { column_magnitudes: BTreeMap::from([( "value".to_string(), diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 7ce5f9a78..0274f3cab 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -91,11 +91,11 @@ pub struct DataWorkload { /// family name; its `metric_type`, `column_signs` and /// `sample_interval` apply to every series of the family /// ([`Self::series_facts`]). Counts, norms and bounds (`series_count`, - /// `stream_norms`, `column_magnitudes`, …) differ per series name (a `foo_bucket` has one - /// series per `le`), so they are read from the series' own entry; - /// pricing a source's own arrival rate reads its `series_count` and - /// `sample_interval` there together. A source without an entry has no - /// known facts. + /// `stream_norms`, `column_magnitudes`, …) differ per series name (a + /// `foo_bucket` has one series per `le`), so they are read from the + /// series' own entry; pricing a source's own arrival rate reads its + /// `series_count` and `sample_interval` there together. A source + /// without an entry has no known facts. #[serde(default)] pub sources: BTreeMap, } @@ -120,9 +120,13 @@ pub struct SourceFacts { pub metric_type: Option, /// The declared sign of each column's values, keyed by column name /// (default: none declared), e.g. `{"value": "non_negative"}` for a - /// latency gauge. A `Scan`'s `column_signs` reads it; an undeclared - /// column is `Unknown`, except a PromQL counter's value, which is - /// non-negative without a declaration. The deployment must ensure it. + /// latency gauge. A `Scan`'s `column_signs` reads it, and so does + /// DDSketch costing, which gives a state over a non-negative column one + /// store instead of two (both by + /// [`declared_sign`](crate::workload::bounds::declared_sign)); an + /// undeclared column is `Unknown`, except a PromQL counter's value, + /// which is non-negative without a declaration. The deployment must + /// ensure it. /// Omitted from JSON when empty. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub column_signs: BTreeMap, diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index bf3e14c3d..8e22cdd5c 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -28,6 +28,7 @@ use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use asap_types::test_support::memo_fixtures::{ aggregate, filter, range, readout, scan, shift, state, }; +use asap_types::workload::bounds::declared_sign; use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; use ValueSign::{NonNegative, NonPositive, Unknown}; @@ -205,6 +206,45 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { } } +/// A `Scan`'s signs are the ones `declared_sign` gives, the rule DDSketch +/// costing reads too: a counter's `value` is non-negative, unless the +/// counter declares another sign for it. Here `requests_total` is a counter +/// with nothing declared and `errors_total` a counter declaring +/// `{"value": "unknown"}`. +#[test] +fn a_scan_signs_its_columns_by_the_declared_sign() { + let data = DataWorkload { + sources: BTreeMap::from([ + ( + "requests_total".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + ..SourceFacts::default() + }, + ), + ( + "errors_total".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + column_signs: BTreeMap::from([("value".to_string(), Unknown)]), + ..SourceFacts::default() + }, + ), + ]), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + for (metric, value) in [("requests_total", NonNegative), ("errors_total", Unknown)] { + let group = add(&mut memo, series(metric)); + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{metric}"); + let declared: Vec<_> = ["ts", "value", "host"] + .into_iter() + .map(|column| declared_sign(&data, metric, column)) + .collect(); + assert_eq!(signs(&memo, group), declared, "{metric}"); + } +} + /// Aggregates: keys keep their sign, sum / max keep the value's, counts are /// non-negative, and a rate is non-negative over a non-negative value only: /// diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 75e2c9bdb..8e5b71c19 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -221,7 +221,7 @@ struct LabelSetDistinctValues { | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | | `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms, bounds and `column_magnitudes` are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | -| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | +| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. DDSketch costing reads it too: a state built from a column its producer passes on unchanged, declared `non_negative` by every source, is costed with one store instead of two. Both read it by one rule (`workload::bounds::declared_sign`): an undeclared column is `unknown`, except a PromQL counter's value, which is `non_negative` unless the counter declares another sign for it. | | `column_magnitudes` | Optional; default none; each `0 < min_abs <= max_abs`, both finite | Bounds on the magnitude of each column's non-zero values, keyed by column name, e.g. `{"value": {"min_abs": 0.001, "max_abs": 10000.0}}` for a latency in seconds. Read per series name: a `histogram` or `summary` family's range does not apply to its `_sum`, `_count` or `_bucket` series. A DDSketch state built from a column its producer passes on unchanged from sources that all declare it is costed at the bins that range spans (over several sources, the widest range among them): the bucket indices from ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one, for γ = (1 + α) / (1 − α), instead of an assumed range of nine decades. The same column's declared `column_signs` decides whether the sketch keeps a negative store. The deployment must ensure the range: one that is too narrow can make Stage 3 pick a finer and much larger sketch than the default range would. Any other value is an input error naming the source and column. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | From cc5bc92d4404683af8e9ed0c7b8faa1081e0c886 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:57:46 +0000 Subject: [PATCH 5/6] test(cost): rebase onto main: the new ProducerOutput in the column-magnitude tests carries no chain width The rebased tests construct ProducerOutput, which main now gives a chain_tumbling_window_width_ms field; a source scan's producer reads from no chain, so None. --- crates/cost/src/node/usage/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 781491e2c..efd664bf0 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -527,6 +527,7 @@ fn ddsketch_state_bytes( family: StateFamily::None, unchanged_columns, tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; let settings = CostInputs::default(); node_resource_usage_and_cost(&build, &[producer_output], (1, 1.0), None, data, &settings) From 68074d0896f005398f2e57dc23deedc576a202c4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 04:05:44 +0000 Subject: [PATCH 6/6] test(cost): rebase onto main: the main-side Hydra/exact state-bytes test's StateInputRows declares no value magnitudes --- crates/cost/src/node/state_bytes/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index 852f6fe52..f9fd83bce 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -516,6 +516,7 @@ fn priced_bytes_agree_with_the_executor() { let input_rows = |rows_per_state: u32| StateInputRows { rows_per_state: Some(u64::from(rows_per_state)), non_negative: true, + value_magnitude_range: None, }; let gap = |issue, priced_over_charged| { Some(KnownBytesGap {