Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 50 additions & 8 deletions crates/accuracy/src/error_contract/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind;
use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue};
use asap_types::ir::schema::FieldDataType;
use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr};
use asap_types::stages::{ColumnLineage, Memo, MemoGroupId};
use asap_types::summary_type::ExactKind;
use asap_types::stages::{ColumnLineage, Memo, MemoGroupId, ValueSign};
use asap_types::summary_type::{ExactKind, SummaryInputExpr};

/// $g_c$'s linear bound for errors in `metric`, each input's and the
/// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the
Expand All @@ -29,18 +29,21 @@ use asap_types::summary_type::ExactKind;
/// - a `Filter` whose predicate reads only columns copied from a source
/// (lineage `SourceColumn`, labels and keys): $\beta = 1$ in any metric;
/// - an `Aggregate` (or exact accumulator build) of counts ($\beta = 0$),
/// min and max (absolute or relative, $\beta = 1$) and avg (absolute,
/// $\beta = 1$), without measure filters or `HAVING`, whose group-by
/// keys are copied from a source as a `Filter`'s predicate columns are
/// (a count by an estimate has no bound);
/// min and max (absolute or relative, $\beta = 1$), avg (absolute,
/// $\beta = 1$) and sum and avg of values of one sign (relative,
/// $\beta = 1$, [`values_of_one_sign`]), without measure filters or
/// `HAVING`, whose group-by keys are copied from a source as a
/// `Filter`'s predicate columns are (a count by an estimate has no
/// bound);
/// - `+` and `-`: absolute, $\beta = 1$; `/`: relative,
/// $\beta = 1/(1-b)$ on both, the denominator capped at $b$; `*`:
/// relative, $(1+b, 1)$, the right operand capped at $b$
/// ($(1+x)(1+y)-1 \le (1+b)\,x + y$ for $y \le b$), with $b$ =
/// [`RELATIVE_OPERAND_VALIDITY_CAP`].
///
/// Every other operator (a filter comparing a computed value, a ranking,
/// a `sum`, a comparison, another function) has none.
/// a `sum` of values of unknown sign or in absolute error, a comparison,
/// another function) has none.
pub(crate) fn error_contract(
memo: &Memo,
consumer: &Operator<MemoGroupId>,
Expand Down Expand Up @@ -80,6 +83,7 @@ pub(crate) fn error_contract(
&& !measures.is_empty()
&& keys_copied_from_a_source(memo, *child, reduction) =>
{
let child_schema = &memo.groups()[child.0 as usize].group_properties.schema;
let mut factor: f64 = 0.0;
for measure in measures {
match measure {
Expand All @@ -88,6 +92,19 @@ pub(crate) fn error_contract(
factor = factor.max(1.0)
}
AggIntent::Avg { .. } if absolute => factor = factor.max(1.0),
AggIntent::Sum { col } | AggIntent::Avg { col }
if metric == AccuracyMetric::Relative
&& col
.or_else(|| {
resolve_column_ref(&ColumnRef::SampleValue, child_schema)
.ok()
})
.is_some_and(|column| {
values_of_one_sign(memo, *child, column)
}) =>
{
factor = factor.max(1.0)
}
_ => return None,
}
}
Expand Down Expand Up @@ -127,20 +144,29 @@ pub(crate) fn error_contract(
| ASAPOp::SummaryMerge { .. } => Some(uniform(1.0)),
ASAPOp::SummaryAgg {
summary_type: FieldDataType::ExactAggregate(kind, _),
input,
filter: None,
reduction,
child,
..
}
| ASAPOp::SummaryTumblingWindowChain {
summary_type: FieldDataType::ExactAggregate(kind, _),
input,
filter: None,
reduction,
child,
..
} if keys_copied_from_a_source(memo, *child, reduction) => match kind {
ExactKind::Count => Some(uniform(0.0)),
ExactKind::Min | ExactKind::Max if extremum_metric => Some(uniform(1.0)),
ExactKind::Sum if metric == AccuracyMetric::Relative => {
let SummaryInputExpr::Column(column) = &input.weight else {
return None;
};
let schema = &memo.groups()[child.0 as usize].group_properties.schema;
let column = resolve_column_ref(column, schema).ok()?;
values_of_one_sign(memo, *child, column).then(|| uniform(1.0))
}
_ => None,
},
_ => None,
Expand All @@ -164,6 +190,22 @@ fn copied_from_a_source(
.all(|column| matches!(lineage.get(column), Some(ColumnLineage::SourceColumn(_))))
}

/// Whether the values of column `column` of `group`'s output all have one
/// sign (`column_signs` `NonNegative` or
/// `NonPositive`, derived through the operators below `group`): then a
/// relative error ε on each value is at most ε on their sum and mean,
/// $|\sum_i (\hat{x}_i - x_i)| \le \varepsilon \sum_i |x_i| =
/// \varepsilon |\sum_i x_i|$. Values of unknown sign may cancel.
fn values_of_one_sign(memo: &Memo, group: MemoGroupId, column: usize) -> bool {
matches!(
memo.groups()[group.0 as usize]
.group_properties
.column_signs
.get(column),
Some(ValueSign::NonNegative | ValueSign::NonPositive)
)
}

/// Whether every key `reduction` groups `child`'s rows by is copied from a
/// source ([`copied_from_a_source`]): `by` keys as listed; a `without`
/// grouping keys by every column of `child` but the time, the sample value
Expand Down
88 changes: 87 additions & 1 deletion crates/accuracy/src/error_contract/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ use asap_types::ir::operator::arguments::Reduction;
use asap_types::ir::scalar::CompareOpKind;
use asap_types::ir::schema::{DataType, PROMQL_SERIES_IDENTITY};
use asap_types::ir::{ExprSemantics, Predicate, ProjectItem};
use asap_types::stages::ValueSign;
use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate};
use asap_types::test_support::memo_fixtures::{aggregate, arithmetic, compare, readout, scan};
use asap_types::workload::DataWorkload;
use asap_types::workload::{DataWorkload, SourceFacts};
use std::collections::BTreeMap;

/// Constant error contracts. A `Filter` whose
/// predicate reads only columns copied from a source keeps or drops
Expand Down Expand Up @@ -285,3 +287,87 @@ fn error_contracts_of_a_ratio_a_count_and_a_sum() {
None
);
}

/// Over values of one sign, a relative error ε on each input is at most
/// ε on their sum and on their mean: $|\sum_i (\hat{x}_i - x_i)| \le
/// \varepsilon \sum_i |x_i| = \varepsilon |\sum_i x_i|$. A `sum` and an
/// `avg` by `job` over a DDSketch 0.99-quantile readout of `latency`,
/// and an exact sum accumulator over it, carry relative error with
/// factor 1 when `latency` is declared non-negative or non-positive (the
/// quantile keeps its sign); undeclared, its values may have both signs
/// and they have none. A `sum` has no absolute contract either way (its
/// errors add up), an `avg` has one with factor 1.
#[test]
fn sums_and_averages_carry_relative_error_over_values_of_one_sign() {
for sign in [
ValueSign::NonNegative,
ValueSign::NonPositive,
ValueSign::Unknown,
] {
let data = DataWorkload {
sources: BTreeMap::from([(
"latency".to_string(),
SourceFacts {
column_signs: BTreeMap::from([("value".to_string(), sign)]),
..SourceFacts::default()
},
)]),
..DataWorkload::default()
};
let mut memo = Memo::new(&data);
let g1 = memo
.find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![]))
.unwrap();
let state = memo
.find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg {
child: g1,
summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch),
input: SummaryUpdate::column(ColumnRef::SampleValue),
reduction: Reduction::by(vec![2]),
filter: None,
}))
.unwrap();
let readout = memo
.find_or_add_group(readout(SketchStatistic::Quantile { q: 0.99 }, state))
.unwrap();
let job = memo.groups()[readout.0 as usize]
.group_properties
.column_lineage
.iter()
.position(|l| matches!(l, ColumnLineage::SourceColumn(_)))
.expect("the readout keeps job");
let by_job = || Reduction::by(vec![job]);
let sum = aggregate(by_job(), AggIntent::Sum { col: None }, readout);
let avg = aggregate(by_job(), AggIntent::Avg { col: None }, readout);
let exact_sum = Operator::ASAP(ASAPOp::SummaryAgg {
child: readout,
summary_type: FieldDataType::exact(ExactKind::Sum),
input: SummaryUpdate::column(ColumnRef::SampleValue),
reduction: by_job(),
filter: None,
});
let factors = |consumer: &Operator<MemoGroupId>, metric| {
error_contract(&memo, consumer, metric).map(|contract| contract.error_factor_per_input)
};
let relative = (sign != ValueSign::Unknown).then(|| vec![1.0]);
for consumer in [&sum, &avg, &exact_sum] {
assert_eq!(
factors(consumer, AccuracyMetric::Relative),
relative,
"{sign:?}: {consumer:?}"
);
assert_eq!(factors(consumer, AccuracyMetric::Rank), None, "{sign:?}");
}
assert_eq!(factors(&sum, AccuracyMetric::ABSOLUTE), None, "{sign:?}");
assert_eq!(
factors(&exact_sum, AccuracyMetric::ABSOLUTE),
None,
"{sign:?}"
);
assert_eq!(
factors(&avg, AccuracyMetric::ABSOLUTE),
Some(vec![1.0]),
"{sign:?}"
);
}
}
68 changes: 61 additions & 7 deletions crates/accuracy/src/metric_rule/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@ use asap_types::ir::operator::AggIntent;
use asap_types::ir::scalar::expr_ir::ArithmeticOpKind;
use asap_types::ir::scalar::{CompareOpKind, ScalarValue};
use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate};
use asap_types::stages::{MemoQuery, RootPruneReason};
use asap_types::stages::{MemoQuery, RootPruneReason, ValueSign};
use asap_types::summary_type::{SketchAlgorithm, SketchStatistic};
use asap_types::test_support::memo_builders::{
estimate, group, own_candidate, quantile_over_time, quantile_over_time_readouts, query,
scan_group, QuantileReadout, QuantileReadouts,
};
use asap_types::test_support::memo_fixtures::{arithmetic, compare, range};
use asap_types::workload::DataWorkload;
use asap_types::workload::{DataWorkload, SourceFacts};
use std::collections::BTreeMap;

fn error_metrics(memo: &Memo) -> ErrorMetricsOfMemo {
ErrorMetricsOfMemo::of(memo, &memo.topological_order().expect("acyclic"))
Expand Down Expand Up @@ -201,11 +202,10 @@ fn a_key_only_filter_keeps_the_ddsketch_readout_admissible() {
assert!(result.is_ok(), "{result:?}");
}

/// `avg` has an absolute contract only here; a relative bound on a
/// mean needs the values' signs. So
/// `avg(quantile_over_time(0.99, m[5m]))` with a relative target cuts
/// the edges from the sketch readouts into the `avg`: it reads the
/// exact quantile only.
/// A relative bound on a mean needs its values' signs, and `m`'s are
/// not declared. So `avg(quantile_over_time(0.99, m[5m]))` with a
/// relative target cuts the edges from the sketch readouts into the
/// `avg`: it reads the exact quantile only.
#[test]
fn avg_of_readouts_under_a_relative_target_reads_exact_only() {
let mut memo = Memo::new(&DataWorkload::default());
Expand Down Expand Up @@ -237,6 +237,60 @@ fn avg_of_readouts_under_a_relative_target_reads_exact_only() {
assert!(result.is_ok(), "{result:?}");
}

/// `avg(quantile_over_time(0.99, m[5m]))` with a relative target, where
/// `m` is declared non-negative or non-positive: a 0.99-quantile of
/// values of one sign has that sign, so the `avg` aggregates values of
/// one sign and carries their relative error with factor 1. The
/// DDSketch readout's edge into the `avg` holds; the KLL readout's
/// rank error is still cut.
#[test]
fn avg_of_readouts_of_values_of_one_sign_keeps_the_ddsketch_edge() {
for sign in [ValueSign::NonNegative, ValueSign::NonPositive] {
let data = DataWorkload {
sources: BTreeMap::from([(
"m".to_string(),
SourceFacts {
column_signs: BTreeMap::from([("value".to_string(), sign)]),
..SourceFacts::default()
},
)]),
..DataWorkload::default()
};
let mut memo = Memo::new(&data);
let p99 = quantile_readouts(&mut memo, 0.99);
let avg = group(
&mut memo,
Operator::NonASAP(NonASAPOp::Aggregate {
child: p99.result,
reduction: Reduction::by(vec![]),
measures: vec![AggIntent::Avg { col: None }],
output_names: vec!["avg".into()],
filters: vec![None],
having: None,
}),
);
let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative);
query(&mut memo, avg, Some(target));
for readout in &p99.by_algorithm {
let result = readout_edge(&memo, readout, own_candidate(avg), 0);
match readout.algorithm {
SketchAlgorithm::DDSketch => assert!(result.is_ok(), "{sign:?}: {result:?}"),
_ => assert!(
matches!(
result,
Err(RootPruneReason::MetricRule {
producer_metric: AccuracyMetric::Rank,
..
})
),
"{sign:?}: {:?} {result:?}",
readout.algorithm
),
}
}
}
}

/// A scalar root's operands are roots: `scalar(v)` returns the value
/// it reads, so `scalar(quantile_over_time(0.99, m[5m]))` at a relative
/// target allows relative error at the quantile group, and the DDSketch
Expand Down
2 changes: 1 addition & 1 deletion crates/accuracy/src/readout_caps/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ fn group_levels(
/// The requirement levels one query gives a readout in `metric` whose cap
/// for it is `cap`; `None` when its target's failure probability has no
/// per-value one. A whole-result δ splits over the values one evaluation
/// returns, $n_q$; a reduction (`max`, `min`, `avg`) fails when any
/// returns, $n_q$; a reduction (`max`, `min`, `avg`, `sum`) fails when any
/// readout it reads does, so δ also splits over the readout group's values
/// per evaluation, `readout_values` (`Memo::max_values_per_evaluation`):
/// the per-value failure probability is
Expand Down
4 changes: 2 additions & 2 deletions docs/design_docs/concepts/accuracy-models.md
Original file line number Diff line number Diff line change
Expand Up @@ -228,11 +228,11 @@ within its validity cap `b_i` (`ErrorContract::error_factor_per_input` and
| Sort, TimeRange, TimeShift, Concat, a readout, a finalize, a summary merge | `β = 1` in any metric |
| Filter on columns copied from a source (labels and keys) | `β = 1` in any metric; a filter comparing a computed value has none |
| Project | Judged on the items reading a computed value (labels, the time and the series identity are copied): `β = 1` in any metric for a copied column; absolute error, `β = 1`, for `abs`, `clamp`, `clamp_min`, `clamp_max` (also PromQL's `promql_` functions) and a negation; relative error, `β = 1`, for a product with or quotient by a nonzero constant |
| Aggregate or exact-accumulator build of counts, min, max, avg | Counts `β = 0`; min and max absolute or relative with `β = 1`; avg absolute with `β = 1`; only when every group-by key is copied from a source (a `by` key as listed; for `without`, every column but the time, the sample value and the excluded ones); none with measure filters or `HAVING`, or with a key reading an estimate |
| Aggregate or exact-accumulator build of counts, min, max, avg, sum | Counts `β = 0`; min and max absolute or relative with `β = 1`; avg absolute with `β = 1`; sum and avg relative with `β = 1` when the values they aggregate are of one sign (the input column's `column_signs` is non-negative or non-positive): `\|Σ (x̂_i − x_i)\| ≤ ε Σ \|x_i\| = ε \|Σ x_i\|`; only when every group-by key is copied from a source (a `by` key as listed; for `without`, every column but the time, the sample value and the excluded ones); none with measure filters or `HAVING`, or with a key reading an estimate |
| `+`, `-` | Absolute error, `β = 1` |
| `/` | Relative error, `β = 1/(1 − b)` on both operands, the denominator capped at `b` |
| `*` | Relative error, `β = (1 + b, 1)`, the right operand capped at `b` |
| Sum, comparison, ranking, counter rate or increase, any other function | No contract: the edge of an approximate input fails |
| Sum in absolute error or of values of unknown sign, comparison, ranking, counter rate or increase, any other function | No contract: the edge of an approximate input fails |
| Top-k over a sketch | Checked: the heap size is at least `k`, and the readout's value guarantee meets the requirement; membership has no check of its own |

`b` is `RELATIVE_OPERAND_VALIDITY_CAP`, 0.1, the weakest error of every
Expand Down
4 changes: 3 additions & 1 deletion docs/develop_docs/end-to-end-accuracy-guarantees.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,15 @@ keeps the metric: an input's error and the output's are in the same one.
| `Aggregate` (or exact accumulator build) by keys copied from a source, without measure filters or `HAVING`: `count` | any | 0 | none |
| the same: `min`, `max` | absolute, relative | 1 | none |
| the same: `avg` | absolute | 1 | none |
| the same: `sum`, `avg` (or exact sum accumulator) of values of one sign (`column_signs` non-negative or non-positive) | relative | 1 | none |
| `+`, `-` | absolute | 1 | none |
| `/` | relative | `1 / (1 - b)` on both | `b` on the denominator |
| `*` | relative | `1 + b` on the left, 1 on the right | `b` on the right operand |

`b` is `RELATIVE_OPERAND_VALIDITY_CAP` (0.1, the weakest error of every
parameter list). Every other operator has no contract: a filter or grouping
on a computed value, a ranking, a `sum`, a comparison, any other function.
on a computed value, a ranking, a `sum` in absolute error or of values of
unknown sign, a comparison, any other function.

The contracts are used in three places:

Expand Down
Loading