diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index 95bd9399f..0ddbd7b7a 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -8,8 +8,8 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue}; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr}; -use asap_types::stages::{ColumnLineage, Memo, MemoGroupId}; -use asap_types::summary_type::ExactKind; +use asap_types::stages::{ColumnLineage, Memo, MemoGroupId, ValueSign}; +use asap_types::summary_type::{ExactKind, SummaryInputExpr}; /// $g_c$'s linear bound for errors in `metric`, each input's and the /// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the @@ -29,10 +29,12 @@ use asap_types::summary_type::ExactKind; /// - a `Filter` whose predicate reads only columns copied from a source /// (lineage `SourceColumn`, labels and keys): $\beta = 1$ in any metric; /// - an `Aggregate` (or exact accumulator build) of counts ($\beta = 0$), -/// min and max (absolute or relative, $\beta = 1$) and avg (absolute, -/// $\beta = 1$), without measure filters or `HAVING`, whose group-by -/// keys are copied from a source as a `Filter`'s predicate columns are -/// (a count by an estimate has no bound); +/// min and max (absolute or relative, $\beta = 1$), avg (absolute, +/// $\beta = 1$) and sum and avg of values of one sign (relative, +/// $\beta = 1$, [`values_of_one_sign`]), without measure filters or +/// `HAVING`, whose group-by keys are copied from a source as a +/// `Filter`'s predicate columns are (a count by an estimate has no +/// bound); /// - `+` and `-`: absolute, $\beta = 1$; `/`: relative, /// $\beta = 1/(1-b)$ on both, the denominator capped at $b$; `*`: /// relative, $(1+b, 1)$, the right operand capped at $b$ @@ -40,7 +42,8 @@ use asap_types::summary_type::ExactKind; /// [`RELATIVE_OPERAND_VALIDITY_CAP`]. /// /// Every other operator (a filter comparing a computed value, a ranking, -/// a `sum`, a comparison, another function) has none. +/// a `sum` of values of unknown sign or in absolute error, a comparison, +/// another function) has none. pub(crate) fn error_contract( memo: &Memo, consumer: &Operator, @@ -80,6 +83,7 @@ pub(crate) fn error_contract( && !measures.is_empty() && keys_copied_from_a_source(memo, *child, reduction) => { + let child_schema = &memo.groups()[child.0 as usize].group_properties.schema; let mut factor: f64 = 0.0; for measure in measures { match measure { @@ -88,6 +92,19 @@ pub(crate) fn error_contract( factor = factor.max(1.0) } AggIntent::Avg { .. } if absolute => factor = factor.max(1.0), + AggIntent::Sum { col } | AggIntent::Avg { col } + if metric == AccuracyMetric::Relative + && col + .or_else(|| { + resolve_column_ref(&ColumnRef::SampleValue, child_schema) + .ok() + }) + .is_some_and(|column| { + values_of_one_sign(memo, *child, column) + }) => + { + factor = factor.max(1.0) + } _ => return None, } } @@ -127,13 +144,14 @@ pub(crate) fn error_contract( | ASAPOp::SummaryMerge { .. } => Some(uniform(1.0)), ASAPOp::SummaryAgg { summary_type: FieldDataType::ExactAggregate(kind, _), + input, filter: None, reduction, child, - .. } | ASAPOp::SummaryTumblingWindowChain { summary_type: FieldDataType::ExactAggregate(kind, _), + input, filter: None, reduction, child, @@ -141,6 +159,14 @@ pub(crate) fn error_contract( } if keys_copied_from_a_source(memo, *child, reduction) => match kind { ExactKind::Count => Some(uniform(0.0)), ExactKind::Min | ExactKind::Max if extremum_metric => Some(uniform(1.0)), + ExactKind::Sum if metric == AccuracyMetric::Relative => { + let SummaryInputExpr::Column(column) = &input.weight else { + return None; + }; + let schema = &memo.groups()[child.0 as usize].group_properties.schema; + let column = resolve_column_ref(column, schema).ok()?; + values_of_one_sign(memo, *child, column).then(|| uniform(1.0)) + } _ => None, }, _ => None, @@ -164,6 +190,22 @@ fn copied_from_a_source( .all(|column| matches!(lineage.get(column), Some(ColumnLineage::SourceColumn(_)))) } +/// Whether the values of column `column` of `group`'s output all have one +/// sign (`column_signs` `NonNegative` or +/// `NonPositive`, derived through the operators below `group`): then a +/// relative error ε on each value is at most ε on their sum and mean, +/// $|\sum_i (\hat{x}_i - x_i)| \le \varepsilon \sum_i |x_i| = +/// \varepsilon |\sum_i x_i|$. Values of unknown sign may cancel. +fn values_of_one_sign(memo: &Memo, group: MemoGroupId, column: usize) -> bool { + matches!( + memo.groups()[group.0 as usize] + .group_properties + .column_signs + .get(column), + Some(ValueSign::NonNegative | ValueSign::NonPositive) + ) +} + /// Whether every key `reduction` groups `child`'s rows by is copied from a /// source ([`copied_from_a_source`]): `by` keys as listed; a `without` /// grouping keys by every column of `child` but the time, the sample value diff --git a/crates/accuracy/src/error_contract/tests.rs b/crates/accuracy/src/error_contract/tests.rs index cd46f06b0..ead745d2b 100644 --- a/crates/accuracy/src/error_contract/tests.rs +++ b/crates/accuracy/src/error_contract/tests.rs @@ -3,9 +3,11 @@ use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::scalar::CompareOpKind; use asap_types::ir::schema::{DataType, PROMQL_SERIES_IDENTITY}; use asap_types::ir::{ExprSemantics, Predicate, ProjectItem}; +use asap_types::stages::ValueSign; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{aggregate, arithmetic, compare, readout, scan}; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, SourceFacts}; +use std::collections::BTreeMap; /// Constant error contracts. A `Filter` whose /// predicate reads only columns copied from a source keeps or drops @@ -285,3 +287,87 @@ fn error_contracts_of_a_ratio_a_count_and_a_sum() { None ); } + +/// Over values of one sign, a relative error ε on each input is at most +/// ε on their sum and on their mean: $|\sum_i (\hat{x}_i - x_i)| \le +/// \varepsilon \sum_i |x_i| = \varepsilon |\sum_i x_i|$. A `sum` and an +/// `avg` by `job` over a DDSketch 0.99-quantile readout of `latency`, +/// and an exact sum accumulator over it, carry relative error with +/// factor 1 when `latency` is declared non-negative or non-positive (the +/// quantile keeps its sign); undeclared, its values may have both signs +/// and they have none. A `sum` has no absolute contract either way (its +/// errors add up), an `avg` has one with factor 1. +#[test] +fn sums_and_averages_carry_relative_error_over_values_of_one_sign() { + for sign in [ + ValueSign::NonNegative, + ValueSign::NonPositive, + ValueSign::Unknown, + ] { + let data = DataWorkload { + sources: BTreeMap::from([( + "latency".to_string(), + SourceFacts { + column_signs: BTreeMap::from([("value".to_string(), sign)]), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let state = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: g1, + summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let readout = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.99 }, state)) + .unwrap(); + let job = memo.groups()[readout.0 as usize] + .group_properties + .column_lineage + .iter() + .position(|l| matches!(l, ColumnLineage::SourceColumn(_))) + .expect("the readout keeps job"); + let by_job = || Reduction::by(vec![job]); + let sum = aggregate(by_job(), AggIntent::Sum { col: None }, readout); + let avg = aggregate(by_job(), AggIntent::Avg { col: None }, readout); + let exact_sum = Operator::ASAP(ASAPOp::SummaryAgg { + child: readout, + summary_type: FieldDataType::exact(ExactKind::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: by_job(), + filter: None, + }); + let factors = |consumer: &Operator, metric| { + error_contract(&memo, consumer, metric).map(|contract| contract.error_factor_per_input) + }; + let relative = (sign != ValueSign::Unknown).then(|| vec![1.0]); + for consumer in [&sum, &avg, &exact_sum] { + assert_eq!( + factors(consumer, AccuracyMetric::Relative), + relative, + "{sign:?}: {consumer:?}" + ); + assert_eq!(factors(consumer, AccuracyMetric::Rank), None, "{sign:?}"); + } + assert_eq!(factors(&sum, AccuracyMetric::ABSOLUTE), None, "{sign:?}"); + assert_eq!( + factors(&exact_sum, AccuracyMetric::ABSOLUTE), + None, + "{sign:?}" + ); + assert_eq!( + factors(&avg, AccuracyMetric::ABSOLUTE), + Some(vec![1.0]), + "{sign:?}" + ); + } +} diff --git a/crates/accuracy/src/metric_rule/tests.rs b/crates/accuracy/src/metric_rule/tests.rs index 991be4279..b76d523bb 100644 --- a/crates/accuracy/src/metric_rule/tests.rs +++ b/crates/accuracy/src/metric_rule/tests.rs @@ -6,14 +6,15 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate}; -use asap_types::stages::{MemoQuery, RootPruneReason}; +use asap_types::stages::{MemoQuery, RootPruneReason, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use asap_types::test_support::memo_builders::{ estimate, group, own_candidate, quantile_over_time, quantile_over_time_readouts, query, scan_group, QuantileReadout, QuantileReadouts, }; use asap_types::test_support::memo_fixtures::{arithmetic, compare, range}; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, SourceFacts}; +use std::collections::BTreeMap; fn error_metrics(memo: &Memo) -> ErrorMetricsOfMemo { ErrorMetricsOfMemo::of(memo, &memo.topological_order().expect("acyclic")) @@ -201,11 +202,10 @@ fn a_key_only_filter_keeps_the_ddsketch_readout_admissible() { assert!(result.is_ok(), "{result:?}"); } -/// `avg` has an absolute contract only here; a relative bound on a -/// mean needs the values' signs. So -/// `avg(quantile_over_time(0.99, m[5m]))` with a relative target cuts -/// the edges from the sketch readouts into the `avg`: it reads the -/// exact quantile only. +/// A relative bound on a mean needs its values' signs, and `m`'s are +/// not declared. So `avg(quantile_over_time(0.99, m[5m]))` with a +/// relative target cuts the edges from the sketch readouts into the +/// `avg`: it reads the exact quantile only. #[test] fn avg_of_readouts_under_a_relative_target_reads_exact_only() { let mut memo = Memo::new(&DataWorkload::default()); @@ -237,6 +237,60 @@ fn avg_of_readouts_under_a_relative_target_reads_exact_only() { assert!(result.is_ok(), "{result:?}"); } +/// `avg(quantile_over_time(0.99, m[5m]))` with a relative target, where +/// `m` is declared non-negative or non-positive: a 0.99-quantile of +/// values of one sign has that sign, so the `avg` aggregates values of +/// one sign and carries their relative error with factor 1. The +/// DDSketch readout's edge into the `avg` holds; the KLL readout's +/// rank error is still cut. +#[test] +fn avg_of_readouts_of_values_of_one_sign_keeps_the_ddsketch_edge() { + for sign in [ValueSign::NonNegative, ValueSign::NonPositive] { + let data = DataWorkload { + sources: BTreeMap::from([( + "m".to_string(), + SourceFacts { + column_signs: BTreeMap::from([("value".to_string(), sign)]), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + let p99 = quantile_readouts(&mut memo, 0.99); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99.result, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, avg, Some(target)); + for readout in &p99.by_algorithm { + let result = readout_edge(&memo, readout, own_candidate(avg), 0); + match readout.algorithm { + SketchAlgorithm::DDSketch => assert!(result.is_ok(), "{sign:?}: {result:?}"), + _ => assert!( + matches!( + result, + Err(RootPruneReason::MetricRule { + producer_metric: AccuracyMetric::Rank, + .. + }) + ), + "{sign:?}: {:?} {result:?}", + readout.algorithm + ), + } + } + } +} + /// A scalar root's operands are roots: `scalar(v)` returns the value /// it reads, so `scalar(quantile_over_time(0.99, m[5m]))` at a relative /// target allows relative error at the quantile group, and the DDSketch diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index 6c8471430..c0784392f 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -162,7 +162,7 @@ fn group_levels( /// The requirement levels one query gives a readout in `metric` whose cap /// for it is `cap`; `None` when its target's failure probability has no /// per-value one. A whole-result δ splits over the values one evaluation -/// returns, $n_q$; a reduction (`max`, `min`, `avg`) fails when any +/// returns, $n_q$; a reduction (`max`, `min`, `avg`, `sum`) fails when any /// readout it reads does, so δ also splits over the readout group's values /// per evaluation, `readout_values` (`Memo::max_values_per_evaluation`): /// the per-value failure probability is diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index d546355fe..f59c6a3f0 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -228,11 +228,11 @@ within its validity cap `b_i` (`ErrorContract::error_factor_per_input` and | Sort, TimeRange, TimeShift, Concat, a readout, a finalize, a summary merge | `β = 1` in any metric | | Filter on columns copied from a source (labels and keys) | `β = 1` in any metric; a filter comparing a computed value has none | | Project | Judged on the items reading a computed value (labels, the time and the series identity are copied): `β = 1` in any metric for a copied column; absolute error, `β = 1`, for `abs`, `clamp`, `clamp_min`, `clamp_max` (also PromQL's `promql_` functions) and a negation; relative error, `β = 1`, for a product with or quotient by a nonzero constant | -| Aggregate or exact-accumulator build of counts, min, max, avg | Counts `β = 0`; min and max absolute or relative with `β = 1`; avg absolute with `β = 1`; only when every group-by key is copied from a source (a `by` key as listed; for `without`, every column but the time, the sample value and the excluded ones); none with measure filters or `HAVING`, or with a key reading an estimate | +| Aggregate or exact-accumulator build of counts, min, max, avg, sum | Counts `β = 0`; min and max absolute or relative with `β = 1`; avg absolute with `β = 1`; sum and avg relative with `β = 1` when the values they aggregate are of one sign (the input column's `column_signs` is non-negative or non-positive): `\|Σ (x̂_i − x_i)\| ≤ ε Σ \|x_i\| = ε \|Σ x_i\|`; only when every group-by key is copied from a source (a `by` key as listed; for `without`, every column but the time, the sample value and the excluded ones); none with measure filters or `HAVING`, or with a key reading an estimate | | `+`, `-` | Absolute error, `β = 1` | | `/` | Relative error, `β = 1/(1 − b)` on both operands, the denominator capped at `b` | | `*` | Relative error, `β = (1 + b, 1)`, the right operand capped at `b` | -| Sum, comparison, ranking, counter rate or increase, any other function | No contract: the edge of an approximate input fails | +| Sum in absolute error or of values of unknown sign, comparison, ranking, counter rate or increase, any other function | No contract: the edge of an approximate input fails | | Top-k over a sketch | Checked: the heap size is at least `k`, and the readout's value guarantee meets the requirement; membership has no check of its own | `b` is `RELATIVE_OPERAND_VALIDITY_CAP`, 0.1, the weakest error of every diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 5cc594dd9..c4676323f 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -119,13 +119,15 @@ keeps the metric: an input's error and the output's are in the same one. | `Aggregate` (or exact accumulator build) by keys copied from a source, without measure filters or `HAVING`: `count` | any | 0 | none | | the same: `min`, `max` | absolute, relative | 1 | none | | the same: `avg` | absolute | 1 | none | +| the same: `sum`, `avg` (or exact sum accumulator) of values of one sign (`column_signs` non-negative or non-positive) | relative | 1 | none | | `+`, `-` | absolute | 1 | none | | `/` | relative | `1 / (1 - b)` on both | `b` on the denominator | | `*` | relative | `1 + b` on the left, 1 on the right | `b` on the right operand | `b` is `RELATIVE_OPERAND_VALIDITY_CAP` (0.1, the weakest error of every parameter list). Every other operator has no contract: a filter or grouping -on a computed value, a ranking, a `sum`, a comparison, any other function. +on a computed value, a ranking, a `sum` in absolute error or of values of +unknown sign, a comparison, any other function. The contracts are used in three places: