Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions crates/accuracy/src/error_contract/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind;
use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue};
use asap_types::ir::schema::FieldDataType;
use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr};
use asap_types::stages::{ColumnLineage, Memo, MemoGroupId, ValueSign};
use asap_types::stages::{ColumnLineage, Memo, MemoGroupId};
use asap_types::summary_type::{ExactKind, SummaryInputExpr};

/// $g_c$'s linear bound for errors in `metric`, each input's and the
Expand Down Expand Up @@ -221,13 +221,11 @@ fn copied_from_a_source(
/// $|\sum_i (\hat{x}_i - x_i)| \le \varepsilon \sum_i |x_i| =
/// \varepsilon |\sum_i x_i|$. Values of unknown sign may cancel.
fn values_of_one_sign(memo: &Memo, group: MemoGroupId, column: usize) -> bool {
matches!(
memo.groups()[group.0 as usize]
.group_properties
.column_signs
.get(column),
Some(ValueSign::NonNegative | ValueSign::NonPositive)
)
memo.groups()[group.0 as usize]
.group_properties
.column_signs
.get(column)
.is_some_and(|sign| sign.is_one_sign())
}

/// Whether every key `reduction` groups `child`'s rows by is copied from a
Expand Down Expand Up @@ -419,7 +417,7 @@ impl ErrorContract {
/// Example: `count` outputs how many values it reads, whatever their
/// error ($\beta = 0$): false; `sum` passes its input's relative error
/// on ($\beta = 1$): true.
pub(crate) fn input_error_reaches_output(&self, input: usize) -> bool {
pub fn input_error_reaches_output(&self, input: usize) -> bool {
self.error_factor_per_input[input] > 0.0
}
}
Expand Down
5 changes: 3 additions & 2 deletions crates/accuracy/src/guarantee/cms/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ fn heap_evaluation_retains_frequency_metric() {
depth: 5,
heap_size: 10,
};
let topk_frequency = guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0)
.expect("heap sketch still provides per-key frequency intervals");
let topk_frequency =
guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0, None, None)
.expect("heap sketch still provides per-key frequency intervals");
assert_eq!(
topk_frequency.spec.accuracy_metric,
AccuracyMetric::ABSOLUTE_STREAM_L1
Expand Down
2 changes: 2 additions & 0 deletions crates/accuracy/src/guarantee/count_sketch/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ fn count_sketch_uses_an_l2_guarantee() {
value: Some("1".into()),
},
1.0,
None,
None,
)
.expect("CountSketch has a parameter-derived L2 guarantee");
assert_eq!(
Expand Down
25 changes: 22 additions & 3 deletions crates/accuracy/src/guarantee/ddsketch.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,32 @@
//! DDSketch relative value error under its estimator/domain contract.
//! DDSketch relative value error under its estimator/domain contract:
//! the bound holds only over values of one sign (`accuracy-models.md`,
//! "Sign conditions"), as the sketch's buckets around 0 have no relative bound.
use super::*;

pub(super) fn guarantee(params: &SketchParams) -> Option<AccuracyGuarantee> {
/// DDSketch's guarantee at `params`: relative error α, failing never when
/// `state_sign`, the sign of the values the state was updated with, is
/// one sign ([`ValueSign::is_one_sign`]); else with an unknown failure
/// probability, so its readout serves no query with an accuracy target.
///
/// Example: `quantile_over_time(0.99, latency[1h])` with `latency`
/// declared non-negative holds at α; over a gauge of unknown sign its
/// failure probability is unknown.
pub(super) fn guarantee(
params: &SketchParams,
state_sign: Option<ValueSign>,
) -> Option<AccuracyGuarantee> {
let SketchParams::DDSketch { alpha } = params else {
return None;
};
let failure_probability = match state_sign.is_some_and(ValueSign::is_one_sign) {
true => ProbabilityExpr::Zero,
false => ProbabilityExpr::Unknown {
statistic: "ddsketch_values_of_unknown_sign".into(),
},
};
Some(super::bounded_guarantee(
AccuracyMetric::Relative,
*alpha,
ProbabilityExpr::Zero,
failure_probability,
))
}
20 changes: 0 additions & 20 deletions crates/accuracy/src/guarantee/hll.rs

This file was deleted.

165 changes: 165 additions & 0 deletions crates/accuracy/src/guarantee/hll/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
//! HLL's distinct count: its relative standard error, without a failure
//! probability, and classic HLL's certified guarantee over a source of at
//! most a declared number of distinct items per evaluation
//! ([`classic_guarantee_at`]), from its linear-counting branch.
//!
//! The certificate assumes independent uniform bucket hashes and an upper
//! bound on the distinct items in the complete evaluation population
//! (including all merged tumbling windows), which the data workload declares
//! (`SourceFacts::max_distinct_per_evaluation`); observations alone do not
//! establish it. It is not an RSE-to-normal conversion or a fit.

use super::*;

/// HLL's guarantee at `params` for its distinct count: classic HLL's
/// certified one at `failure_probability` when the source declares at most
/// `max_distinct_per_evaluation` distinct items and the contract covers that
/// bound ([`classic_guarantee_at`]); else the relative standard error, whose
/// failure probability is unknown.
pub(super) fn guarantee(
params: &SketchParams,
failure_probability: f64,
max_distinct_per_evaluation: Option<u64>,
) -> Option<AccuracyGuarantee> {
let SketchParams::Hll { precision, .. } = params else {
return None;
};
max_distinct_per_evaluation
.and_then(|bound| classic_guarantee_at(*precision, failure_probability, bound))
.or_else(|| {
Some(super::bounded_guarantee(
AccuracyMetric::Relative,
relative_standard_error(*precision),
ProbabilityExpr::Unknown {
statistic: "hll_estimator_failure_probability".into(),
},
))
})
}

/// HLL's relative standard error, `1.04 / √m` for `m = 2^precision` buckets.
fn relative_standard_error(precision: u8) -> f64 {
1.04 / 2f64.powi(i32::from(precision)).sqrt()
}

/// Classic HLL's guarantee at `precision` over a source of at most
/// `max_distinct` distinct items per evaluation: the least relative error
/// whose failure probability ([`ClassicHllConfidence`]) is at most
/// `failure_probability`, with that failure probability. `None` when no
/// relative error below 1 is certified (the bound is outside the contract
/// for this precision).
///
/// Example: at precision 18 and δ = 0.01, a bound of 9,636 certifies a
/// relative error just under 0.05; a bound of 1,000,000 certifies none.
fn classic_guarantee_at(
precision: u8,
failure_probability: f64,
max_distinct: u64,
) -> Option<AccuracyGuarantee> {
let max_distinct = u32::try_from(max_distinct).ok()?;
let certified = |relative_error: f64| {
ClassicHllConfidence::new(max_distinct, relative_error)
.and_then(|model| model.failure_probability(precision))
.is_some_and(|delta| delta <= failure_probability)
};
// The failure probability falls as the relative error grows: bisect
// for the least certified one, keeping the certified end.
let (mut uncertified, mut certified_error) = (1e-6, 1.0 - 1e-6);
if !certified(certified_error) {
return None;
}
for _ in 0..60 {
let middle = (uncertified + certified_error) / 2.0;
match certified(middle) {
true => certified_error = middle,
false => uncertified = middle,
}
}
let delta =
ClassicHllConfidence::new(max_distinct, certified_error)?.failure_probability(precision)?;
Some(super::bounded_guarantee(
AccuracyMetric::Relative,
certified_error,
ProbabilityExpr::Constant { value: delta },
))
}

/// The largest precision classic HLL's contract covers.
const MAX_PRECISION: u8 = 18;

/// The largest distinct bound the contract supports at any precision:
/// `m/2` at precision 18. Evaluating a bound is O(bound).
const MAX_CONTRACT_DISTINCT: u32 = 1 << 17;

/// A finite-population contract for `m * ln(m / zero_registers)` with the
/// classic HLL small-range switch, over at most `max_distinct` distinct
/// items and at relative error `relative_error`. Hashing is assumed
/// independent and uniform.
#[derive(Debug, Clone, Copy)]
struct ClassicHllConfidence {
max_distinct: u32,
relative_error: f64,
}

impl ClassicHllConfidence {
/// `None` for a bound of 0 or above [`MAX_CONTRACT_DISTINCT`], or a
/// relative error outside [1e-6, 1).
fn new(max_distinct: u32, relative_error: f64) -> Option<Self> {
(max_distinct > 0
&& max_distinct <= MAX_CONTRACT_DISTINCT
&& relative_error.is_finite()
&& (1e-6..1.0).contains(&relative_error))
.then_some(Self {
max_distinct,
relative_error,
})
}

/// The probability that the estimate at `precision` is off by more
/// than the relative error, for any population up to the bound; `None`
/// outside the contract (a precision outside 4..=18, a bound above
/// `m/2`, or a relative error the deterministic overestimate exceeds).
///
/// A finite bound, not an asymptotic RSE fit. With N distinct hashes
/// and K occupied buckets, C = N − K collision arrivals satisfy
/// P(C ≥ t) ≤ λ^t / t!, λ = N(N−1)/(2m): each arrival's conditional
/// collision probability is at most (i−1)/m, and a union bound over t
/// arrivals is bounded by the t-th power of their sum divided by t!.
///
/// N ≤ m/2 makes the classic raw estimate ≤ 2 α_m m < 2.5 m, so the
/// small-range switch always uses L = −m ln(1 − K/m). Then
/// K ≤ L ≤ N + N²/(2(m − N)). The latter bounds overestimation
/// deterministically; underestimation implies C > ε N. The collision
/// bound is maximized over every integer N in the contract, not just
/// its upper end (small populations' tails matter).
fn failure_probability(&self, precision: u8) -> Option<f64> {
if !(4..=MAX_PRECISION).contains(&precision) {
return None;
}
let m = f64::from(1u32 << precision);
let max_n = f64::from(self.max_distinct);
// Reserve numerical slack; do not certify sub-floating-point error.
let eps = self.relative_error - 1e-8;
if max_n > m / 2.0 || max_n / (2.0 * (m - max_n)) > eps {
return None;
}
let mut log_factorial = vec![0.0; self.max_distinct as usize + 1];
for i in 1..log_factorial.len() {
log_factorial[i] = log_factorial[i - 1] + (i as f64).ln();
}
let mut worst = 0.0_f64;
for n in 2..=self.max_distinct {
let nf = f64::from(n);
// Including a boundary collision event is conservative.
let t = ((eps * nf).floor() as usize + 1).min(n as usize);
let lambda = nf * (nf - 1.0) / (2.0 * m);
let log_tail = (t as f64) * lambda.ln() - log_factorial[t];
worst = worst.max(log_tail.min(0.0).exp());
}
// Never return a spurious zero from underflow or cancellation.
Some((worst * (1.0 + 1e-10) + 1e-12).min(1.0))
}
}

#[cfg(test)]
mod tests;
96 changes: 96 additions & 0 deletions crates/accuracy/src/guarantee/hll/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
use super::*;

/// The failure probability at a relative error, over a bound, at a
/// precision; 1 outside the contract.
fn classic_failure_probability(precision: u8, relative_error: f64, max_distinct: u64) -> f64 {
u32::try_from(max_distinct)
.ok()
.and_then(|n| ClassicHllConfidence::new(n, relative_error))
.and_then(|model| model.failure_probability(precision))
.unwrap_or(1.0)
}

/// The largest distinct bound certified at δ = 0.01 for a target ε, by
/// precision; a bound outside the contract certifies nothing.
#[test]
fn largest_certified_bounds_by_precision() {
for (eps, precision, largest) in [
(0.05, 15, 1176),
(0.05, 18, 9636),
(0.02, 17, 1842),
(0.02, 18, 3791),
(0.01, 18, 72),
] {
assert!(
classic_failure_probability(precision, eps, largest) <= 0.01,
"eps={eps} p={precision} n={largest}"
);
assert!(
classic_failure_probability(precision, eps, largest + 1) > 0.01,
"eps={eps} p={precision} n={}",
largest + 1
);
}
for too_large in [u64::from(MAX_CONTRACT_DISTINCT) + 1, u64::MAX] {
assert_eq!(classic_failure_probability(18, 0.05, too_large), 1.0);
}
}

/// Exact occupancy probabilities independently check both tails for every
/// population up to the bound.
#[test]
fn probability_bound_dominates_exact_occupancy_distribution() {
for precision in 4..=10 {
let m = 1usize << precision;
let max_n = 64.min(m / 2);
for eps in [0.05, 0.2, 0.6] {
let model = ClassicHllConfidence::new(max_n as u32, eps).unwrap();
let Some(bound) = model.failure_probability(precision) else {
continue;
};
let mut occupancy = vec![0.0; max_n + 1];
occupancy[0] = 1.0;
for n in 1..=max_n {
let mut next = vec![0.0; max_n + 1];
for k in 0..n {
next[k] += occupancy[k] * k as f64 / m as f64;
next[k + 1] += occupancy[k] * (m - k) as f64 / m as f64;
}
occupancy = next;
let actual: f64 = occupancy
.iter()
.enumerate()
.filter_map(|(k, &prob)| {
let estimate = -(m as f64) * (-(k as f64) / (m as f64)).ln_1p();
((estimate - n as f64).abs() > eps * n as f64).then_some(prob)
})
.sum();
assert!(
actual <= bound + 1e-12,
"p={precision} n={n} eps={eps}: {actual}>{bound}"
);
}
}
}
}

/// `COUNT(DISTINCT user_id)` within relative 0.05 at δ = 0.01 from classic
/// HLL at precision 18 is certified over a source of at most 9,636
/// distinct users per evaluation; over 1,000,000 (outside the contract) and
/// without a bound only the relative standard error is stated, whose
/// failure probability is unknown.
#[test]
fn a_distinct_count_is_certified_over_the_source_s_distinct_bound() {
let precision_18 = SketchParams::Hll { precision: 18 };
let at = |bound| guarantee(&precision_18, 0.01, bound).unwrap();
let certified = at(Some(9_636));
assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::Relative);
let error = certified.spec.error.evaluate().unwrap();
assert!(error <= 0.05 && error > 0.04, "{error}");
assert!(certified.spec.failure_probability.evaluate().unwrap() <= 0.01);
for uncertified in [Some(1_000_000), None] {
let guarantee = at(uncertified);
assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative);
assert_eq!(guarantee.spec.failure_probability.evaluate(), None);
}
}
Loading
Loading