diff --git a/crates/cost/src/node/state_bytes/mod.rs b/crates/cost/src/node/state_bytes/mod.rs index 6f039cfea..fac0cc11e 100644 --- a/crates/cost/src/node/state_bytes/mod.rs +++ b/crates/cost/src/node/state_bytes/mod.rs @@ -6,6 +6,7 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::summary_type::KLL_MAX_K; use asap_types::summary_type::{SketchParams, EXACT_ACCUMULATOR_STATE_BYTES}; +use asap_types::workload::ValueMagnitudeRange; /// Plain values are 8 bytes, strings 16; summary columns are sized apart. pub fn row_bytes(schema: &Schema) -> u64 { @@ -43,8 +44,12 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// `ASSUMED_VALUE_KEY_BYTES`; /// - DDSketch and KMV: the rows the state folds, at least one (any number /// when unknown), each a distinct item for KMV; a DDSketch's values span -/// `DDSKETCH_ASSUMED_VALUE_RANGE` and fill its negative store too -/// unless proven non-negative. +/// `input_rows.value_magnitude_range` when declared, else +/// `DDSKETCH_ASSUMED_MAGNITUDE_RANGE`, and fill its negative store too +/// unless proven non-negative. Its bins are the bucket indices from +/// ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one, for γ = (1 + α) / +/// (1 - α); this DDSketch has no bin limit to cap them. E.g. α = 0.01 +/// over [0.001, 10000] takes indices -346 to 460, 807 bins. /// /// KLL is priced without the library's estimate: its fixed bytes /// (`kll_fixed_bytes`: the struct and its level array) plus the 8-byte @@ -76,8 +81,8 @@ pub fn summary_shape( } /// What one summary state's bytes depend on besides its parameters (only -/// a KLL's, a DDSketch's and a KMV's do): the input rows it folds and the -/// sign of their values. +/// a KLL's, a DDSketch's and a KMV's do): the input rows it folds, the +/// sign of their values and, for a DDSketch, their magnitudes. /// The rows are whatever its producer emits (raw data samples, rates, sums), /// not a summary sample. #[derive(Debug, Clone, Copy, PartialEq)] @@ -86,9 +91,15 @@ pub struct StateInputRows { /// `None` when unknown, which sizes it for every value it can be assumed /// to see. pub rows_per_state: Option, - /// Whether every value folded is proven non-negative (raw data samples of - /// sources declared non-negative), so a DDSketch fills no negative store. + /// Whether every value folded is proven non-negative (a source column + /// passed on unchanged, which every source declares non-negative), so a + /// DDSketch fills no negative store. pub non_negative: bool, + /// Bounds on `|value|` of the non-zero values folded into the state; + /// `None` when not declared, which costs a DDSketch over the assumed + /// `DDSKETCH_ASSUMED_MAGNITUDE_RANGE`. E.g. `Some(ValueMagnitudeRange { + /// min_abs: 0.001, max_abs: 10000.0 })` for a latency in seconds. + pub value_magnitude_range: Option, } impl StateInputRows { @@ -96,6 +107,7 @@ impl StateInputRows { pub const UNKNOWN: Self = Self { rows_per_state: None, non_negative: false, + value_magnitude_range: None, }; } @@ -160,16 +172,21 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, SketchParams::DDSketch { alpha } if !(*alpha > 0.0 && *alpha < 1.0) => { (1.0, std::mem::size_of::() as u64) } - SketchParams::DDSketch { alpha } => ( - 1.0, - DdSketch::estimated_memory_bytes( - *alpha, - rows, + SketchParams::DDSketch { alpha } => { + let ValueMagnitudeRange { min_abs, max_abs } = input_rows + .value_magnitude_range + .unwrap_or(DDSKETCH_ASSUMED_MAGNITUDE_RANGE); + ( 1.0, - DDSKETCH_ASSUMED_VALUE_RANGE, - !input_rows.non_negative, - ) as u64, - ), + DdSketch::estimated_memory_bytes( + *alpha, + rows, + min_abs, + max_abs, + !input_rows.non_negative, + ) as u64, + ) + } SketchParams::Hll { precision, .. } => ( 1.0, HyperLogLog::::estimated_memory_bytes(usize::from(*precision)) as u64, @@ -270,12 +287,14 @@ fn univmon_expected_layers(layers: u8) -> f64 { 2.0 - 2f64.powi(1 - i32::from(layers.max(1))) } -/// Ratio of the largest to the smallest value magnitude a DDSketch is -/// assumed to see: nine decades, e.g. latencies from 1 ns to 1 s, priced as -/// magnitudes in `[1, 1e9]` (its bins depend on the ratio alone). An -/// assumption, not workload evidence: the data workload declares no value -/// range. -const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; +/// The range of value magnitudes a DDSketch is assumed to see when its +/// column declares no `column_magnitudes`: nine decades, e.g. latencies from 1 ns +/// to 1 s, costed as magnitudes in `[1, 1e9]` (its bins depend on the ratio +/// alone). An assumption, not workload evidence. +const DDSKETCH_ASSUMED_MAGNITUDE_RANGE: ValueMagnitudeRange = ValueMagnitudeRange { + min_abs: 1.0, + max_abs: 1e9, +}; #[cfg(test)] mod tests; diff --git a/crates/cost/src/node/state_bytes/tests.rs b/crates/cost/src/node/state_bytes/tests.rs index add5ba6fe..f9fd83bce 100644 --- a/crates/cost/src/node/state_bytes/tests.rs +++ b/crates/cost/src/node/state_bytes/tests.rs @@ -108,6 +108,7 @@ fn ddsketch_state_bytes_bound_the_library() { StateInputRows { rows_per_state, non_negative, + value_magnitude_range: None, }, ) .1 @@ -145,6 +146,41 @@ fn ddsketch_state_bytes_bound_the_library() { assert_eq!(priced(Some(10_000), false), priced(None, false)); } +/// A DDSketch over a column declaring its value magnitudes is costed at +/// the bins they span, not nine decades. At α = 0.01, γ = 1.01 / 0.99 and +/// a value `v` falls in bin `floor(ln v / ln γ)`: magnitudes in +/// [0.001, 10000] take bins -346 to 460, 807 bins, where the default +/// [1, 1e9] takes 0 to 1036, 1,037. With three or more non-negative rows +/// the library's bound is `2 (bins + 126)` 8-byte slots plus the struct. +#[test] +fn ddsketch_state_bytes_follow_the_declared_value_magnitude_range() { + let alpha = 0.01; + let costed = |value_magnitude_range| { + sketch_shape( + &SketchParams::DDSketch { alpha }, + StateInputRows { + rows_per_state: Some(600), + non_negative: true, + value_magnitude_range, + }, + ) + .1 + }; + let struct_bytes = std::mem::size_of::() as u64; + let bytes_for_bins = |bins: u64| struct_bytes + 8 * 2 * (bins + 126); + let latency = ValueMagnitudeRange { + min_abs: 0.001, + max_abs: 10_000.0, + }; + assert_eq!(costed(Some(latency)), bytes_for_bins(807)); + assert_eq!(costed(None), bytes_for_bins(1_037)); + // Values a narrower range holds fit the costed bytes. + let values: Vec = (0..600) + .map(|i| 10f64.powf(-3.0 + 7.0 * f64::from(i) / 599.0)) + .collect(); + assert!(library_ddsketch_bytes(alpha, &values) <= costed(Some(latency))); +} + /// A KLL state is charged what `asap_sketchlib` allocates for the /// samples it sees, not its full capacity: its fixed struct and level /// array, plus the item array the library starts empty and doubles from @@ -168,6 +204,7 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { StateInputRows { rows_per_state, non_negative: true, + value_magnitude_range: None, }, ) .1 @@ -197,7 +234,7 @@ fn kll_state_bytes_grow_with_the_samples_it_sees() { /// Plan selection prices each state the executor builds per group at /// what the executor charges for it (#947). Each family is built /// through the executor's factory and fed `n` rows shaped as the -/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_VALUE_RANGE`], +/// pricing assumes: signed values over [`DDSKETCH_ASSUMED_MAGNITUDE_RANGE`], /// label values of [`ASSUMED_VALUE_KEY_BYTES`] keys for UnivMon, or /// keyed rows whose tuple keys are [`ASSUMED_HEAP_KEY_BYTES`] long. The /// heap families' shapes are dominated by their heaps. The charge is @@ -366,7 +403,7 @@ fn state_bytes_bound_the_executors_charge() { // scrambled order, alternating in sign. Rows::Values => { let at = f64::from(i * 7_919 % n) / f64::from(n); - let magnitude = DDSKETCH_ASSUMED_VALUE_RANGE.powf(at); + let magnitude = DDSKETCH_ASSUMED_MAGNITUDE_RANGE.max_abs.powf(at); let sign = if i % 2 == 0 { 1.0 } else { -1.0 }; updater.update_single(sign * magnitude, 0).unwrap(); } @@ -384,6 +421,7 @@ fn state_bytes_bound_the_executors_charge() { StateInputRows { rows_per_state: Some(u64::from(n)), non_negative: false, + value_magnitude_range: None, }, ) .1 as usize; @@ -478,6 +516,7 @@ fn priced_bytes_agree_with_the_executor() { let input_rows = |rows_per_state: u32| StateInputRows { rows_per_state: Some(u64::from(rows_per_state)), non_negative: true, + value_magnitude_range: None, }; let gap = |issue, priced_over_charged| { Some(KnownBytesGap { diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 5288d954f..efc2bd383 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -35,16 +35,17 @@ use crate::{ }; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::resolve_column_ref; use asap_types::ir::schema::{FieldDataType, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; use asap_types::stages::{Lifetime, StorageMedium}; -use asap_types::summary_type::{SketchParams, SketchStatistic}; +use asap_types::summary_type::{SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate}; use asap_types::workload::bounds::{ - declared_non_negative, declared_series, group_count_bound, source_name, source_rows_per_ms, - unchanged_source_columns, + declared_non_negative, declared_series, declared_value_magnitude_range, group_count_bound, + source_name, source_rows_per_ms, unchanged_source_columns, }; use asap_types::workload::{DataWorkload, ONE_SECOND_MS}; @@ -182,6 +183,22 @@ pub struct ProducerOutput<'a> { pub chain_tumbling_window_width_ms: Option, } +impl ProducerOutput<'_> { + /// The source column `weight` reads, when this producer passes that + /// column on unchanged from the sources: the column whose values a + /// summary built over this producer folds. E.g. `value` for a DDSketch + /// over a scan of `latency` or over a filter on that scan; `None` for a + /// computed weight such as a rate. + pub(crate) fn unchanged_column(&self, weight: &SummaryInputExpr) -> Option<&str> { + let SummaryInputExpr::Column(column) = weight else { + return None; + }; + let index = resolve_column_ref(column, self.output_schema).ok()?; + let name = self.output_schema.fields.get(index)?.name.as_str(); + self.unchanged_columns.contains(name).then_some(name) + } +} + /// What one node uses and outputs ([`node_resource_usage_and_cost`]): its /// output statistics, sources and state family for its consumers; its work /// for one computation (one execution) in ms on one core; its usage per @@ -481,18 +498,22 @@ pub fn node_resource_usage_and_cost<'a>( .ok_or(AnalyticalCostError::Overflow)?, }) }; - // Whether a summary built here sees raw data samples (its producer a - // scan or a time range) of sources all declared non-negative. - let raw_data_samples_non_negative = || { - let raw = producers.first().is_some_and(|producer| { - matches!( - producer.payload, - Payload::NonASAP(NonASAPOp::Scan { .. } | NonASAPOp::TimeRange { .. }) - ) - }); - raw && sources - .iter() - .all(|source| declared_non_negative(data, source)) + // What the sources declare about the values a summary built here folds: + // whether every one is non-negative, and their magnitude range, both + // from the one source column it folds + // (`ProducerOutput::unchanged_column`). + let folded_values = |update: &SummaryUpdate| { + producers + .first() + .and_then(|producer| producer.unchanged_column(&update.weight)) + .map_or((false, None), |column| { + let non_negative = !sources.is_empty() + && sources + .iter() + .all(|source| declared_non_negative(data, source, column)); + let range = declared_value_magnitude_range(data, sources.iter().copied(), column); + (non_negative, range) + }) }; let unary = |output| UnaryEdgeStatistics { input, @@ -807,6 +828,7 @@ pub fn node_resource_usage_and_cost<'a>( }, Payload::ASAP(ASAPOp::SummaryAgg { summary_type: family, + input: update, reduction, .. }) => { @@ -814,9 +836,11 @@ pub fn node_resource_usage_and_cost<'a>( // At query time, each state sees its group's input rows; at // ingestion time, the rows of a window one second of // arriving rows does not bound. + let (non_negative, value_magnitude_range) = folded_values(update); let input_rows = StateInputRows { rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), + non_negative, + value_magnitude_range, }; let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); @@ -842,6 +866,7 @@ pub fn node_resource_usage_and_cost<'a>( } Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type: family, + input: update, reduction, tumbling_window_width_ms, .. @@ -859,9 +884,11 @@ pub fn node_resource_usage_and_cost<'a>( .map_err(|_| AnalyticalCostError::Overflow)?, false => input.rows.div_ceil(retained_output_count), }; + let (non_negative, value_magnitude_range) = folded_values(update); let input_rows = StateInputRows { rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), + non_negative, + value_magnitude_range, }; let (depth, state_bytes) = summary_shape(family, node.parameters.as_ref(), input_rows)?; let state_bytes = measured_state_bytes.unwrap_or(state_bytes); diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 98fadfad6..efd664bf0 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -486,3 +486,252 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { .unwrap(); assert_eq!(usage.peak_memory_bytes, 16_000 * 66); } + +/// The bytes of a DDSketch state at α = 0.01 built over `column` of the +/// 600 rows `producer` emits from `source`, which passes on +/// `unchanged_columns` unchanged. +fn ddsketch_state_bytes( + data: &DataWorkload, + producer: Payload, + source: &str, + column: &str, + unchanged_columns: BTreeSet<&str>, +) -> u64 { + use asap_types::ir::schema::{DataType, Field}; + use asap_types::summary_type::SketchAlgorithm; + let producer = PhysicalASAPDAGNode { + output_schema: Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain(column, DataType::Float64, false), + ]), + ..node(producer) + }; + let build = PhysicalASAPDAGNode { + parameters: Some(SketchParams::DDSketch { alpha: 0.01 }), + ..node(Payload::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), + input: SummaryUpdate::column(ColumnRef::Named(column.to_string())), + reduction: Reduction::Reduce(GroupKeys::default()), + filter: None, + })) + }; + let producer_output = ProducerOutput { + payload: &producer.payload, + output_schema: &producer.output_schema, + output: EdgeStatistics { + rows: 600, + bytes: 9_600, + }, + sources: BTreeSet::from([source]), + family: StateFamily::None, + unchanged_columns, + tumbling_windows: 1, + chain_tumbling_window_width_ms: None, + }; + let settings = CostInputs::default(); + node_resource_usage_and_cost(&build, &[producer_output], (1, 1.0), None, data, &settings) + .unwrap() + .output + .bytes +} + +/// The bytes `sketch_shape` gives a DDSketch state at α = 0.01 over 600 +/// rows of the given sign and magnitude range. +fn ddsketch_costed_bytes( + non_negative: bool, + value_magnitude_range: Option, +) -> u64 { + use crate::node::state_bytes::{sketch_shape, StateInputRows}; + sketch_shape( + &SketchParams::DDSketch { alpha: 0.01 }, + StateInputRows { + rows_per_state: Some(600), + non_negative, + value_magnitude_range, + }, + ) + .1 +} + +/// A scan of PromQL metric `metric`. +fn metric_scan(metric: &str) -> Payload { + Payload::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::default(), + }) +} + +/// A data workload whose one source `source` declares `facts`. +fn declaring(source: &str, facts: asap_types::workload::SourceFacts) -> DataWorkload { + DataWorkload { + sources: BTreeMap::from([(source.to_string(), facts)]), + ..DataWorkload::default() + } +} + +/// A DDSketch built over a scan's `value` is costed at the bins its +/// source's declared `column_magnitudes` span: 807 at α = 0.01 for +/// [0.001, 10000], against 1,037 for the assumed [1, 1e9]. Undeclared, or +/// a column its producer does not pass on unchanged, keeps the assumed +/// range. +#[test] +fn a_ddsketch_over_a_scan_is_costed_at_its_declared_value_magnitude_range() { + use asap_types::workload::{SourceFacts, ValueMagnitudeRange}; + let latency = ValueMagnitudeRange { + min_abs: 0.001, + max_abs: 10_000.0, + }; + let data = |range: Option| { + declaring( + "latency", + SourceFacts { + column_magnitudes: range + .map(|range| ("value".to_string(), range)) + .into_iter() + .collect(), + ..SourceFacts::default() + }, + ) + }; + let state_bytes = |data: &DataWorkload, unchanged_columns| { + ddsketch_state_bytes( + data, + metric_scan("latency"), + "latency", + "value", + unchanged_columns, + ) + }; + let unchanged = || BTreeSet::from(["ts", "value"]); + assert_eq!( + state_bytes(&data(Some(latency)), unchanged()), + ddsketch_costed_bytes(false, Some(latency)) + ); + assert_eq!( + state_bytes(&data(None), unchanged()), + ddsketch_costed_bytes(false, None) + ); + assert_eq!( + state_bytes(&data(Some(latency)), BTreeSet::from(["ts"])), + ddsketch_costed_bytes(false, None) + ); + assert!(ddsketch_costed_bytes(false, Some(latency)) < ddsketch_costed_bytes(false, None)); +} + +/// A DDSketch over a SQL table's `latency` column that the table declares +/// non-negative keeps one store: the sign is read for the folded column, +/// not for a PromQL `value`. Undeclared, it keeps both stores. +#[test] +fn a_ddsketch_over_a_sql_column_declared_non_negative_keeps_one_store() { + use asap_types::stages::ValueSign; + use asap_types::workload::SourceFacts; + let table_scan = || { + Payload::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "requests".into(), + }, + predicates: vec![], + schema: Schema::default(), + }) + }; + let declared = declaring( + "requests", + SourceFacts { + column_signs: BTreeMap::from([("latency".to_string(), ValueSign::NonNegative)]), + ..SourceFacts::default() + }, + ); + let state_bytes = |data: &DataWorkload| { + ddsketch_state_bytes( + data, + table_scan(), + "requests", + "latency", + BTreeSet::from(["ts", "latency"]), + ) + }; + assert_eq!(state_bytes(&declared), ddsketch_costed_bytes(true, None)); + assert_eq!( + state_bytes(&DataWorkload::default()), + ddsketch_costed_bytes(false, None) + ); + assert!(ddsketch_costed_bytes(true, None) < ddsketch_costed_bytes(false, None)); +} + +/// A DDSketch over a filter on a scan of a source declared non-negative +/// keeps one store: the filter passes `value` on unchanged. A producer +/// that computes `value` (not among its unchanged columns) keeps both. +#[test] +fn a_ddsketch_over_a_filter_on_a_non_negative_scan_keeps_one_store() { + use asap_types::ir::scalar::{Predicate, ScalarExpr, ScalarValue}; + use asap_types::stages::ValueSign; + use asap_types::workload::SourceFacts; + let filter = || { + Payload::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + child: 0, + }) + }; + let data = declaring( + "latency", + SourceFacts { + column_signs: BTreeMap::from([("value".to_string(), ValueSign::NonNegative)]), + ..SourceFacts::default() + }, + ); + let state_bytes = |unchanged_columns| { + ddsketch_state_bytes(&data, filter(), "latency", "value", unchanged_columns) + }; + assert_eq!( + state_bytes(BTreeSet::from(["ts", "value"])), + ddsketch_costed_bytes(true, None) + ); + assert_eq!( + state_bytes(BTreeSet::from(["ts"])), + ddsketch_costed_bytes(false, None) + ); +} + +/// A counter's `value` is non-negative unless the source declares another +/// sign for it: a counter `requests_total` declaring `{"value": "unknown"}` +/// keeps both stores, the sign the memo gives its `Scan` too. Without the +/// declaration it keeps one store. +#[test] +fn a_ddsketch_over_a_counter_declaring_its_value_unknown_keeps_both_stores() { + use asap_types::stages::ValueSign; + use asap_types::workload::{PrometheusMetricType, SourceFacts}; + let counter = |column_signs| { + declaring( + "requests_total", + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + column_signs, + ..SourceFacts::default() + }, + ) + }; + let state_bytes = |data: &DataWorkload| { + ddsketch_state_bytes( + data, + metric_scan("requests_total"), + "requests_total", + "value", + BTreeSet::from(["ts", "value"]), + ) + }; + assert_eq!( + state_bytes(&counter(BTreeMap::from([( + "value".to_string(), + ValueSign::Unknown + )]))), + ddsketch_costed_bytes(false, None) + ); + assert_eq!( + state_bytes(&counter(BTreeMap::new())), + ddsketch_costed_bytes(true, None) + ); +} diff --git a/crates/frontend-promql/tests/histogram_metadata.rs b/crates/frontend-promql/tests/histogram_metadata.rs index 1fa25858f..8b44fbcea 100644 --- a/crates/frontend-promql/tests/histogram_metadata.rs +++ b/crates/frontend-promql/tests/histogram_metadata.rs @@ -183,9 +183,9 @@ fn declared_summary_family() { Summary )); let data = over_family("", "foo", Summary).data_workload.unwrap(); - assert!(declared_non_negative(&data, "foo_sum")); - assert!(declared_non_negative(&data, "foo_count")); - assert!(!declared_non_negative(&data, "foo")); + assert!(declared_non_negative(&data, "foo_sum", "value")); + assert!(declared_non_negative(&data, "foo_count", "value")); + assert!(!declared_non_negative(&data, "foo", "value")); } /// A declared histogram family `foo`: `foo_sum` and `foo_count` read as @@ -203,6 +203,6 @@ fn declared_histogram_family() { Histogram )); let data = over_family("", "foo", Histogram).data_workload.unwrap(); - assert!(declared_non_negative(&data, "foo_sum")); - assert!(declared_non_negative(&data, "foo_count")); + assert!(declared_non_negative(&data, "foo_sum", "value")); + assert!(declared_non_negative(&data, "foo_count", "value")); } diff --git a/crates/types/src/stages/logical/group/column_properties.rs b/crates/types/src/stages/logical/group/column_properties.rs index 8bab8f34d..b3dd1dd6d 100644 --- a/crates/types/src/stages/logical/group/column_properties.rs +++ b/crates/types/src/stages/logical/group/column_properties.rs @@ -26,8 +26,8 @@ use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, }; -use crate::workload::bounds::source_name; -use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; +use crate::workload::bounds::{declared_column_sign, source_name}; +use crate::workload::{series_facts, SourceFacts}; /// The sign of one column's values, a group property derived from the /// input groups like lineage (concepts doc, section 3, "Value sign"); a @@ -201,26 +201,15 @@ fn non_asap_columns( use NonASAPOp::*; match op { // Each column copied from its source, with the sign its source's - // facts declare (a histogram's or summary's series reads its - // family's); a counter's `value` is non-negative without a - // declaration. + // facts declare (`declared_sign`): e.g. a counter's `value` is + // non-negative unless it declares another sign. Scan { source, schema, .. } => { let facts = series_facts(source_facts, source_name(source)); schema .fields .iter() .map(|field| ColumnProperties { - sign: match facts { - Some((facts, series_type)) => facts - .column_signs - .get(&field.name) - .copied() - .or((field.name == "value" - && series_type == Some(PrometheusMetricType::Counter)) - .then_some(NonNegative)) - .unwrap_or(Unknown), - None => Unknown, - }, + sign: declared_column_sign(facts, &field.name), lineage: Some(ColumnLineage::SourceColumn(SourceColumn { source: source.clone(), scan_ordinal: 0, diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 4b17f2150..32bff6d3b 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -11,8 +11,9 @@ use crate::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, ScalarExpr, }; use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; +use crate::stages::ValueSign; use crate::summary_type::SketchStatistic; -use crate::workload::{DataWorkload, PrometheusMetricType}; +use crate::workload::{DataWorkload, PrometheusMetricType, SourceFacts, ValueMagnitudeRange}; /// A scan source's name: its PromQL metric or SQL table. pub fn source_name(source: &Source) -> &str { @@ -405,23 +406,72 @@ fn at_most_one_row(node: &OperatorNode) -> bool { } } -/// Whether `data` declares every sample of `metric` non-negative: the -/// metric is declared a counter, or its `value` column is declared -/// non-negative ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). -pub fn declared_non_negative(data: &DataWorkload, metric: &str) -> bool { - declared_counter(data, metric) - || data.series_facts(metric).is_some_and(|(facts, _)| { - facts.column_signs.get("value") == Some(&crate::stages::ValueSign::NonNegative) - }) +/// The sign `data` declares for `source`'s `column`: its +/// [`SourceFacts::column_signs`] entry (a histogram or summary family's +/// applies to its series, [`DataWorkload::series_facts`]), else +/// `NonNegative` for a counter's `value`, else `Unknown`. The memo signs a +/// `Scan`'s columns by this rule, and DDSketch costing reads it. E.g. a +/// counter `requests_total` gives `NonNegative` for `value`, or `Unknown` +/// when it declares `{"value": "unknown"}`; a SQL table declaring +/// `{"latency": "non_negative"}` gives `NonNegative` for `latency`. +pub fn declared_sign(data: &DataWorkload, source: &str, column: &str) -> ValueSign { + declared_column_sign(data.series_facts(source), column) } -/// Whether `data` declares `metric` a counter, itself or as the `_sum` or -/// `_count` series of a declared histogram or summary -/// ([`DataWorkload::series_facts`]). A declared type does not expire, so its -/// evidence is not checked. -fn declared_counter(data: &DataWorkload, metric: &str) -> bool { - data.series_facts(metric) - .is_some_and(|(_, series_type)| series_type == Some(PrometheusMetricType::Counter)) +/// [`declared_sign`] from the facts and series type +/// [`DataWorkload::series_facts`] finds for the source, for a caller that +/// holds only the sources: `Unknown` when there are none. +pub(crate) fn declared_column_sign( + series_facts: Option<(&SourceFacts, Option)>, + column: &str, +) -> ValueSign { + let Some((facts, series_type)) = series_facts else { + return ValueSign::Unknown; + }; + match facts.column_signs.get(column) { + Some(&declared) => declared, + None if column == "value" && series_type == Some(PrometheusMetricType::Counter) => { + ValueSign::NonNegative + } + None => ValueSign::Unknown, + } +} + +/// Whether `data` declares every value of `source`'s `column` non-negative +/// ([`declared_sign`]). +pub fn declared_non_negative(data: &DataWorkload, source: &str, column: &str) -> bool { + declared_sign(data, source, column) == ValueSign::NonNegative +} + +/// The range of magnitudes of `column`'s non-zero values over every source +/// in `sources`, from each one's own declared `column_magnitudes` +/// ([`SourceFacts::column_magnitudes`](crate::workload::SourceFacts::column_magnitudes)): +/// the smallest `min_abs` and the largest `max_abs` among them. Each series +/// is read under its own name, so a histogram family `foo`'s range does not +/// bound `foo_sum`, `foo_count` or `foo_bucket`. `None` when `sources` is +/// empty or some source declares none for `column`. E.g. sources declaring +/// `[0.001, 10]` and `[0.01, 100]` give `[0.001, 100]`. +pub fn declared_value_magnitude_range<'a>( + data: &DataWorkload, + sources: impl IntoIterator, + column: &str, +) -> Option { + let declared = |source: &str| { + data.sources + .get(source)? + .column_magnitudes + .get(column) + .copied() + }; + let mut sources = sources.into_iter(); + let first = declared(sources.next()?)?; + sources.try_fold(first, |union, source| { + let range = declared(source)?; + Some(ValueMagnitudeRange { + min_abs: union.min_abs.min(range.min_abs), + max_abs: union.max_abs.max(range.max_abs), + }) + }) } /// Rows per millisecond the sources in `sources` receive together, each @@ -679,6 +729,84 @@ mod tests { use super::*; use crate::workload::{DataArrival, DurationMs, SourceFacts}; + /// The magnitudes over several sources span all of theirs, and are + /// unknown when any of them declares none for the column. + #[test] + fn declared_value_magnitude_ranges_are_the_union_over_the_sources() { + let facts = |min_abs, max_abs| SourceFacts { + column_magnitudes: BTreeMap::from([( + "value".to_string(), + ValueMagnitudeRange { min_abs, max_abs }, + )]), + ..SourceFacts::default() + }; + let data = DataWorkload { + sources: BTreeMap::from([ + ("a".to_string(), facts(0.001, 10.0)), + ("b".to_string(), facts(0.01, 100.0)), + ("c".to_string(), SourceFacts::default()), + ]), + ..DataWorkload::default() + }; + let magnitude = |sources: &[&str], column| { + declared_value_magnitude_range(&data, sources.iter().copied(), column) + }; + assert_eq!( + magnitude(&["a", "b"], "value"), + Some(ValueMagnitudeRange { + min_abs: 0.001, + max_abs: 100.0 + }) + ); + assert_eq!(magnitude(&["a", "c"], "value"), None); + assert_eq!(magnitude(&["a"], "latency"), None); + assert_eq!(magnitude(&[], "value"), None); + } + + /// A histogram family's declared range bounds its observations, not + /// the counts in `foo_count` or the sums in `foo_sum`: each series' + /// range is read from its own entry. Here `foo` declares observations + /// in [0.001, 10] and `foo_count` its counts in [1, 1e6]. + #[test] + fn a_series_magnitudes_come_from_its_own_entry_not_its_family() { + let magnitudes = |min_abs, max_abs| { + BTreeMap::from([( + "value".to_string(), + ValueMagnitudeRange { min_abs, max_abs }, + )]) + }; + let data = DataWorkload { + sources: BTreeMap::from([ + ( + "foo".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Histogram), + column_magnitudes: magnitudes(0.001, 10.0), + ..SourceFacts::default() + }, + ), + ( + "foo_count".to_string(), + SourceFacts { + column_magnitudes: magnitudes(1.0, 1e6), + ..SourceFacts::default() + }, + ), + ]), + ..DataWorkload::default() + }; + let magnitude = |source| declared_value_magnitude_range(&data, [source], "value"); + assert_eq!( + magnitude("foo_count"), + Some(ValueMagnitudeRange { + min_abs: 1.0, + max_abs: 1e6 + }) + ); + assert_eq!(magnitude("foo_sum"), None); + assert_eq!(magnitude("foo_bucket"), None); + } + /// A source's rows per ms: its series over its sample interval when it /// declares both, else its series share of the workload's rate. #[test] diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 93385c36e..0274f3cab 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -91,11 +91,11 @@ pub struct DataWorkload { /// family name; its `metric_type`, `column_signs` and /// `sample_interval` apply to every series of the family /// ([`Self::series_facts`]). Counts, norms and bounds (`series_count`, - /// `stream_norms`, …) differ per series name (a `foo_bucket` has one - /// series per `le`), so they are read from the series' own entry; - /// pricing a source's own arrival rate reads its `series_count` and - /// `sample_interval` there together. A source without an entry has no - /// known facts. + /// `stream_norms`, `column_magnitudes`, …) differ per series name (a + /// `foo_bucket` has one series per `le`), so they are read from the + /// series' own entry; pricing a source's own arrival rate reads its + /// `series_count` and `sample_interval` there together. A source + /// without an entry has no known facts. #[serde(default)] pub sources: BTreeMap, } @@ -120,12 +120,29 @@ pub struct SourceFacts { pub metric_type: Option, /// The declared sign of each column's values, keyed by column name /// (default: none declared), e.g. `{"value": "non_negative"}` for a - /// latency gauge. A `Scan`'s `column_signs` reads it; an undeclared - /// column is `Unknown`, except a PromQL counter's value, which is - /// non-negative without a declaration. The deployment must ensure it. + /// latency gauge. A `Scan`'s `column_signs` reads it, and so does + /// DDSketch costing, which gives a state over a non-negative column one + /// store instead of two (both by + /// [`declared_sign`](crate::workload::bounds::declared_sign)); an + /// undeclared column is `Unknown`, except a PromQL counter's value, + /// which is non-negative without a declaration. The deployment must + /// ensure it. /// Omitted from JSON when empty. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub column_signs: BTreeMap, + /// Bounds on the magnitude of each column's non-zero values, keyed by + /// column name (default: none declared), e.g. `{"value": {"min_abs": + /// 0.001, "max_abs": 10000.0}}` for a latency in seconds. Read per + /// series name: a histogram or summary family's range does not apply to + /// its `_sum`, `_count` or `_bucket` series. A DDSketch state built from + /// the column unchanged is costed at the bins those magnitudes span (the + /// bucket indices from ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one) + /// instead of an assumed range of nine decades. The deployment must + /// ensure it: a range that is too narrow can make Stage 3 pick a finer + /// and much larger sketch than the default range would. Omitted from + /// JSON when empty. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub column_magnitudes: BTreeMap, /// How often each series of the metric (or group of rows of a SQL /// table) receives a sample (default: unknown). A PromQL or MetricsQL /// bare instant selector over the metric reads the latest sample within it; lowering such a @@ -196,6 +213,19 @@ pub struct SourceFacts { pub rows_held: Option, } +/// Bounds on the magnitude of one column's non-zero values: +/// `min_abs <= |v| <= max_abs` for every non-zero `v`, with +/// `0 < min_abs <= max_abs`, both finite. E.g. `{"min_abs": 0.001, +/// "max_abs": 10000.0}` for latencies from 1 ms to about 3 h in seconds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ValueMagnitudeRange { + /// The smallest magnitude of a non-zero value. + pub min_abs: f64, + /// The largest magnitude of a value. + pub max_abs: f64, +} + /// The distinct combinations of values a set of labels (or columns) of one /// source takes together, an upper bound. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -372,6 +402,21 @@ impl DataWorkload { } } } + for (column, magnitude) in &facts.column_magnitudes { + let ValueMagnitudeRange { min_abs, max_abs } = *magnitude; + if !(min_abs.is_finite() + && max_abs.is_finite() + && 0.0 < min_abs + && min_abs <= max_abs) + { + return Err(WorkloadError::InvalidValueMagnitudeRange { + source_name: source_name.clone(), + column: column.clone(), + min_abs, + max_abs, + }); + } + } if facts.max_distinct_per_evaluation == Some(0) { return Err(WorkloadError::ZeroMaxDistinct); } @@ -507,6 +552,46 @@ mod tests { assert_eq!(with(0).validate(), Err(WorkloadError::ZeroMaxDistinct)); } + /// A declared value magnitude has `0 < min_abs <= max_abs`, both + /// finite; any other is an input error naming the source and column. + #[test] + fn column_magnitudes_must_be_positive_ordered_and_finite() { + let with = |min_abs: f64, max_abs: f64| DataWorkload { + sources: BTreeMap::from([( + "latency".to_string(), + SourceFacts { + column_magnitudes: BTreeMap::from([( + "value".to_string(), + ValueMagnitudeRange { min_abs, max_abs }, + )]), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(0.001, 10_000.0).validate().is_ok()); + assert!(with(5.0, 5.0).validate().is_ok()); + for (min_abs, max_abs) in [ + (0.0, 1.0), + (-1.0, 1.0), + (10.0, 1.0), + (1.0, f64::INFINITY), + (f64::NAN, 1.0), + (1.0, f64::NAN), + ] { + let error = with(min_abs, max_abs).validate().unwrap_err(); + assert!( + matches!( + &error, + WorkloadError::InvalidValueMagnitudeRange { source_name, column, .. } + if source_name == "latency" && column == "value" + ), + "{min_abs}, {max_abs}: {error:?}" + ); + assert!(error.to_string().contains("min_abs"), "{error}"); + } + } + /// A declared `rows_held` is positive. #[test] fn rows_held_must_be_positive() { diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index cff20a85c..96dd88390 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -11,7 +11,7 @@ mod query; pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, - StreamNorms, ONE_SECOND_MS, + StreamNorms, ValueMagnitudeRange, ONE_SECOND_MS, }; pub use planning::{PlanningWorkload, WorkloadError}; pub use query::{ diff --git a/crates/types/src/workload/planning.rs b/crates/types/src/workload/planning.rs index 4ab90a894..cf735ce22 100644 --- a/crates/types/src/workload/planning.rs +++ b/crates/types/src/workload/planning.rs @@ -134,6 +134,18 @@ pub enum WorkloadError { distinct_values: u64, label_product: u64, }, + /// Bad input: a `column_magnitudes` entry that is not + /// `0 < min_abs <= max_abs` with both finite. + #[error( + "source {source_name}: column_magnitudes for column {column} must have \ + 0 < min_abs <= max_abs, both finite, got min_abs {min_abs}, max_abs {max_abs}" + )] + InvalidValueMagnitudeRange { + source_name: String, + column: String, + min_abs: f64, + max_abs: f64, + }, /// Bad input: a source's `rows_held` of zero. #[error("a source's rows_held must be greater than zero")] ZeroRowsHeld, @@ -187,6 +199,13 @@ mod tests { "value".to_string(), ValueSign::NonNegative, )]), + column_magnitudes: BTreeMap::from([( + "value".to_string(), + ValueMagnitudeRange { + min_abs: 1.0, + max_abs: 1e12, + }, + )]), sample_interval: Some(DurationMs(15_000)), stream_norms: Some(StreamNorms { l1: Some(1e9), @@ -216,6 +235,7 @@ mod tests { "series_count": 10000, "metric_type": "counter", "column_signs": {"value": "non_negative"}, + "column_magnitudes": {"value": {"min_abs": 1.0, "max_abs": 1e12}}, "sample_interval": 15000, "stream_norms": {"l1": 1e9, "l2": null}, "max_distinct_per_evaluation": 100, @@ -247,5 +267,8 @@ mod tests { ..SourceFacts::default() } ); + // Empty column magnitudes are omitted from JSON. + let json = serde_json::to_value(&minimal.sources["flows"]).unwrap(); + assert!(json.get("column_magnitudes").is_none(), "{json}"); } } diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index bf3e14c3d..8e22cdd5c 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -28,6 +28,7 @@ use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use asap_types::test_support::memo_fixtures::{ aggregate, filter, range, readout, scan, shift, state, }; +use asap_types::workload::bounds::declared_sign; use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; use ValueSign::{NonNegative, NonPositive, Unknown}; @@ -205,6 +206,45 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { } } +/// A `Scan`'s signs are the ones `declared_sign` gives, the rule DDSketch +/// costing reads too: a counter's `value` is non-negative, unless the +/// counter declares another sign for it. Here `requests_total` is a counter +/// with nothing declared and `errors_total` a counter declaring +/// `{"value": "unknown"}`. +#[test] +fn a_scan_signs_its_columns_by_the_declared_sign() { + let data = DataWorkload { + sources: BTreeMap::from([ + ( + "requests_total".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + ..SourceFacts::default() + }, + ), + ( + "errors_total".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + column_signs: BTreeMap::from([("value".to_string(), Unknown)]), + ..SourceFacts::default() + }, + ), + ]), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + for (metric, value) in [("requests_total", NonNegative), ("errors_total", Unknown)] { + let group = add(&mut memo, series(metric)); + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{metric}"); + let declared: Vec<_> = ["ts", "value", "host"] + .into_iter() + .map(|column| declared_sign(&data, metric, column)) + .collect(); + assert_eq!(signs(&memo, group), declared, "{metric}"); + } +} + /// Aggregates: keys keep their sign, sum / max keep the value's, counts are /// non-negative, and a rate is non-negative over a non-negative value only: /// diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 16348633b..8e5b71c19 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -197,6 +197,7 @@ struct SourceFacts { series_count: Option, metric_type: Option, column_signs: BTreeMap, + column_magnitudes: BTreeMap, sample_interval: Option, stream_norms: Option, max_distinct_per_evaluation: Option, @@ -204,6 +205,10 @@ struct SourceFacts { distinct_values_per_label_set: Vec, rows_held: Option, } +struct ValueMagnitudeRange { + min_abs: f64, + max_abs: f64, +} struct LabelSetDistinctValues { labels: Vec, distinct_values: u64, @@ -215,8 +220,9 @@ struct LabelSetDistinctValues { | `arrival` | Present; may be `Unknown` | Distinguishes data at rest, ingesting data, and mixed data. Ingestion-time builds are possible only for ingesting or mixed data. | | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | -| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | -| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | +| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms, bounds and `column_magnitudes` are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | +| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. DDSketch costing reads it too: a state built from a column its producer passes on unchanged, declared `non_negative` by every source, is costed with one store instead of two. Both read it by one rule (`workload::bounds::declared_sign`): an undeclared column is `unknown`, except a PromQL counter's value, which is `non_negative` unless the counter declares another sign for it. | +| `column_magnitudes` | Optional; default none; each `0 < min_abs <= max_abs`, both finite | Bounds on the magnitude of each column's non-zero values, keyed by column name, e.g. `{"value": {"min_abs": 0.001, "max_abs": 10000.0}}` for a latency in seconds. Read per series name: a `histogram` or `summary` family's range does not apply to its `_sum`, `_count` or `_bucket` series. A DDSketch state built from a column its producer passes on unchanged from sources that all declare it is costed at the bins that range spans (over several sources, the widest range among them): the bucket indices from ⌊log_γ min_abs⌋ to ⌊log_γ max_abs⌋, at least one, for γ = (1 + α) / (1 − α), instead of an assumed range of nine decades. The same column's declared `column_signs` decides whether the sketch keeps a negative store. The deployment must ensure the range: one that is too narrow can make Stage 3 pick a finer and much larger sketch than the default range would. Any other value is an input error naming the source and column. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 126c6615d..1932537c6 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -306,7 +306,10 @@ query's window comes from its text (range and offset). These facts are separate: `DataWorkload` holds `arrival` (`DataArrival`: `AtRest`, `Ingesting` or `Mixed` with `rows_per_second`, or `Unknown`) and `sources`, a map from PromQL metric or SQL table name to `SourceFacts`: `series_count`, -`metric_type`, `column_signs`, `sample_interval` (PromQL and +`metric_type`, `column_signs`, `column_magnitudes` (bounds on each +column's non-zero value magnitudes, `{"value": {"min_abs": 0.001, +"max_abs": 10000.0}}`, read per series name, which cost a DDSketch state +at the bins that range spans instead of nine decades), `sample_interval` (PromQL and MetricsQL), `stream_norms`, `max_distinct_per_evaluation` and `distinct_values_per_label` (distinct values per label or column, which plan selection prices a `by (...)` grouping's groups at: the product of its