diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index 97b16bf63..c4517dd32 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -8,7 +8,7 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue}; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr}; -use asap_types::stages::{ColumnLineage, Memo, MemoGroupId, ValueSign}; +use asap_types::stages::{ColumnLineage, Memo, MemoGroupId}; use asap_types::summary_type::{ExactKind, SummaryInputExpr}; /// $g_c$'s linear bound for errors in `metric`, each input's and the @@ -221,13 +221,11 @@ fn copied_from_a_source( /// $|\sum_i (\hat{x}_i - x_i)| \le \varepsilon \sum_i |x_i| = /// \varepsilon |\sum_i x_i|$. Values of unknown sign may cancel. fn values_of_one_sign(memo: &Memo, group: MemoGroupId, column: usize) -> bool { - matches!( - memo.groups()[group.0 as usize] - .group_properties - .column_signs - .get(column), - Some(ValueSign::NonNegative | ValueSign::NonPositive) - ) + memo.groups()[group.0 as usize] + .group_properties + .column_signs + .get(column) + .is_some_and(|sign| sign.is_one_sign()) } /// Whether every key `reduction` groups `child`'s rows by is copied from a @@ -419,7 +417,7 @@ impl ErrorContract { /// Example: `count` outputs how many values it reads, whatever their /// error ($\beta = 0$): false; `sum` passes its input's relative error /// on ($\beta = 1$): true. - pub(crate) fn input_error_reaches_output(&self, input: usize) -> bool { + pub fn input_error_reaches_output(&self, input: usize) -> bool { self.error_factor_per_input[input] > 0.0 } } diff --git a/crates/accuracy/src/guarantee/cms/tests.rs b/crates/accuracy/src/guarantee/cms/tests.rs index 12b35605f..7087f2a0b 100644 --- a/crates/accuracy/src/guarantee/cms/tests.rs +++ b/crates/accuracy/src/guarantee/cms/tests.rs @@ -7,8 +7,9 @@ fn heap_evaluation_retains_frequency_metric() { depth: 5, heap_size: 10, }; - let topk_frequency = guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0) - .expect("heap sketch still provides per-key frequency intervals"); + let topk_frequency = + guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0, None, None) + .expect("heap sketch still provides per-key frequency intervals"); assert_eq!( topk_frequency.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1 diff --git a/crates/accuracy/src/guarantee/count_sketch/tests.rs b/crates/accuracy/src/guarantee/count_sketch/tests.rs index f0505631f..190df3858 100644 --- a/crates/accuracy/src/guarantee/count_sketch/tests.rs +++ b/crates/accuracy/src/guarantee/count_sketch/tests.rs @@ -36,6 +36,8 @@ fn count_sketch_uses_an_l2_guarantee() { value: Some("1".into()), }, 1.0, + None, + None, ) .expect("CountSketch has a parameter-derived L2 guarantee"); assert_eq!( diff --git a/crates/accuracy/src/guarantee/ddsketch.rs b/crates/accuracy/src/guarantee/ddsketch.rs index fde3d390f..30cfdab75 100644 --- a/crates/accuracy/src/guarantee/ddsketch.rs +++ b/crates/accuracy/src/guarantee/ddsketch.rs @@ -1,13 +1,32 @@ -//! DDSketch relative value error under its estimator/domain contract. +//! DDSketch relative value error under its estimator/domain contract: +//! the bound holds only over values of one sign (`accuracy-models.md`, +//! "Sign conditions"), as the sketch's buckets around 0 have no relative bound. use super::*; -pub(super) fn guarantee(params: &SketchParams) -> Option { +/// DDSketch's guarantee at `params`: relative error α, failing never when +/// `state_sign`, the sign of the values the state was updated with, is +/// one sign ([`ValueSign::is_one_sign`]); else with an unknown failure +/// probability, so its readout serves no query with an accuracy target. +/// +/// Example: `quantile_over_time(0.99, latency[1h])` with `latency` +/// declared non-negative holds at α; over a gauge of unknown sign its +/// failure probability is unknown. +pub(super) fn guarantee( + params: &SketchParams, + state_sign: Option, +) -> Option { let SketchParams::DDSketch { alpha } = params else { return None; }; + let failure_probability = match state_sign.is_some_and(ValueSign::is_one_sign) { + true => ProbabilityExpr::Zero, + false => ProbabilityExpr::Unknown { + statistic: "ddsketch_values_of_unknown_sign".into(), + }, + }; Some(super::bounded_guarantee( AccuracyMetric::Relative, *alpha, - ProbabilityExpr::Zero, + failure_probability, )) } diff --git a/crates/accuracy/src/guarantee/hll.rs b/crates/accuracy/src/guarantee/hll.rs deleted file mode 100644 index ef83b3e67..000000000 --- a/crates/accuracy/src/guarantee/hll.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! HLL's relative standard error, without a failure probability. - -use super::*; - -pub(super) fn guarantee(params: &SketchParams) -> Option { - let SketchParams::Hll { precision, .. } = params else { - return None; - }; - Some(super::bounded_guarantee( - AccuracyMetric::Relative, - relative_standard_error(*precision), - ProbabilityExpr::Unknown { - statistic: "hll_estimator_failure_probability".into(), - }, - )) -} -/// HLL's relative standard error, `1.04 / √m` for `m = 2^precision` buckets. -fn relative_standard_error(precision: u8) -> f64 { - 1.04 / 2f64.powi(i32::from(precision)).sqrt() -} diff --git a/crates/accuracy/src/guarantee/hll/mod.rs b/crates/accuracy/src/guarantee/hll/mod.rs new file mode 100644 index 000000000..62db84dc7 --- /dev/null +++ b/crates/accuracy/src/guarantee/hll/mod.rs @@ -0,0 +1,165 @@ +//! HLL's distinct count: its relative standard error, without a failure +//! probability, and classic HLL's certified guarantee over a source of at +//! most a declared number of distinct items per evaluation +//! ([`classic_guarantee_at`]), from its linear-counting branch. +//! +//! The certificate assumes independent uniform bucket hashes and an upper +//! bound on the distinct items in the complete evaluation population +//! (including all merged tumbling windows), which the data workload declares +//! (`SourceFacts::max_distinct_per_evaluation`); observations alone do not +//! establish it. It is not an RSE-to-normal conversion or a fit. + +use super::*; + +/// HLL's guarantee at `params` for its distinct count: classic HLL's +/// certified one at `failure_probability` when the source declares at most +/// `max_distinct_per_evaluation` distinct items and the contract covers that +/// bound ([`classic_guarantee_at`]); else the relative standard error, whose +/// failure probability is unknown. +pub(super) fn guarantee( + params: &SketchParams, + failure_probability: f64, + max_distinct_per_evaluation: Option, +) -> Option { + let SketchParams::Hll { precision, .. } = params else { + return None; + }; + max_distinct_per_evaluation + .and_then(|bound| classic_guarantee_at(*precision, failure_probability, bound)) + .or_else(|| { + Some(super::bounded_guarantee( + AccuracyMetric::Relative, + relative_standard_error(*precision), + ProbabilityExpr::Unknown { + statistic: "hll_estimator_failure_probability".into(), + }, + )) + }) +} + +/// HLL's relative standard error, `1.04 / √m` for `m = 2^precision` buckets. +fn relative_standard_error(precision: u8) -> f64 { + 1.04 / 2f64.powi(i32::from(precision)).sqrt() +} + +/// Classic HLL's guarantee at `precision` over a source of at most +/// `max_distinct` distinct items per evaluation: the least relative error +/// whose failure probability ([`ClassicHllConfidence`]) is at most +/// `failure_probability`, with that failure probability. `None` when no +/// relative error below 1 is certified (the bound is outside the contract +/// for this precision). +/// +/// Example: at precision 18 and δ = 0.01, a bound of 9,636 certifies a +/// relative error just under 0.05; a bound of 1,000,000 certifies none. +fn classic_guarantee_at( + precision: u8, + failure_probability: f64, + max_distinct: u64, +) -> Option { + let max_distinct = u32::try_from(max_distinct).ok()?; + let certified = |relative_error: f64| { + ClassicHllConfidence::new(max_distinct, relative_error) + .and_then(|model| model.failure_probability(precision)) + .is_some_and(|delta| delta <= failure_probability) + }; + // The failure probability falls as the relative error grows: bisect + // for the least certified one, keeping the certified end. + let (mut uncertified, mut certified_error) = (1e-6, 1.0 - 1e-6); + if !certified(certified_error) { + return None; + } + for _ in 0..60 { + let middle = (uncertified + certified_error) / 2.0; + match certified(middle) { + true => certified_error = middle, + false => uncertified = middle, + } + } + let delta = + ClassicHllConfidence::new(max_distinct, certified_error)?.failure_probability(precision)?; + Some(super::bounded_guarantee( + AccuracyMetric::Relative, + certified_error, + ProbabilityExpr::Constant { value: delta }, + )) +} + +/// The largest precision classic HLL's contract covers. +const MAX_PRECISION: u8 = 18; + +/// The largest distinct bound the contract supports at any precision: +/// `m/2` at precision 18. Evaluating a bound is O(bound). +const MAX_CONTRACT_DISTINCT: u32 = 1 << 17; + +/// A finite-population contract for `m * ln(m / zero_registers)` with the +/// classic HLL small-range switch, over at most `max_distinct` distinct +/// items and at relative error `relative_error`. Hashing is assumed +/// independent and uniform. +#[derive(Debug, Clone, Copy)] +struct ClassicHllConfidence { + max_distinct: u32, + relative_error: f64, +} + +impl ClassicHllConfidence { + /// `None` for a bound of 0 or above [`MAX_CONTRACT_DISTINCT`], or a + /// relative error outside [1e-6, 1). + fn new(max_distinct: u32, relative_error: f64) -> Option { + (max_distinct > 0 + && max_distinct <= MAX_CONTRACT_DISTINCT + && relative_error.is_finite() + && (1e-6..1.0).contains(&relative_error)) + .then_some(Self { + max_distinct, + relative_error, + }) + } + + /// The probability that the estimate at `precision` is off by more + /// than the relative error, for any population up to the bound; `None` + /// outside the contract (a precision outside 4..=18, a bound above + /// `m/2`, or a relative error the deterministic overestimate exceeds). + /// + /// A finite bound, not an asymptotic RSE fit. With N distinct hashes + /// and K occupied buckets, C = N − K collision arrivals satisfy + /// P(C ≥ t) ≤ λ^t / t!, λ = N(N−1)/(2m): each arrival's conditional + /// collision probability is at most (i−1)/m, and a union bound over t + /// arrivals is bounded by the t-th power of their sum divided by t!. + /// + /// N ≤ m/2 makes the classic raw estimate ≤ 2 α_m m < 2.5 m, so the + /// small-range switch always uses L = −m ln(1 − K/m). Then + /// K ≤ L ≤ N + N²/(2(m − N)). The latter bounds overestimation + /// deterministically; underestimation implies C > ε N. The collision + /// bound is maximized over every integer N in the contract, not just + /// its upper end (small populations' tails matter). + fn failure_probability(&self, precision: u8) -> Option { + if !(4..=MAX_PRECISION).contains(&precision) { + return None; + } + let m = f64::from(1u32 << precision); + let max_n = f64::from(self.max_distinct); + // Reserve numerical slack; do not certify sub-floating-point error. + let eps = self.relative_error - 1e-8; + if max_n > m / 2.0 || max_n / (2.0 * (m - max_n)) > eps { + return None; + } + let mut log_factorial = vec![0.0; self.max_distinct as usize + 1]; + for i in 1..log_factorial.len() { + log_factorial[i] = log_factorial[i - 1] + (i as f64).ln(); + } + let mut worst = 0.0_f64; + for n in 2..=self.max_distinct { + let nf = f64::from(n); + // Including a boundary collision event is conservative. + let t = ((eps * nf).floor() as usize + 1).min(n as usize); + let lambda = nf * (nf - 1.0) / (2.0 * m); + let log_tail = (t as f64) * lambda.ln() - log_factorial[t]; + worst = worst.max(log_tail.min(0.0).exp()); + } + // Never return a spurious zero from underflow or cancellation. + Some((worst * (1.0 + 1e-10) + 1e-12).min(1.0)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/hll/tests.rs b/crates/accuracy/src/guarantee/hll/tests.rs new file mode 100644 index 000000000..6f60bd8a4 --- /dev/null +++ b/crates/accuracy/src/guarantee/hll/tests.rs @@ -0,0 +1,96 @@ +use super::*; + +/// The failure probability at a relative error, over a bound, at a +/// precision; 1 outside the contract. +fn classic_failure_probability(precision: u8, relative_error: f64, max_distinct: u64) -> f64 { + u32::try_from(max_distinct) + .ok() + .and_then(|n| ClassicHllConfidence::new(n, relative_error)) + .and_then(|model| model.failure_probability(precision)) + .unwrap_or(1.0) +} + +/// The largest distinct bound certified at δ = 0.01 for a target ε, by +/// precision; a bound outside the contract certifies nothing. +#[test] +fn largest_certified_bounds_by_precision() { + for (eps, precision, largest) in [ + (0.05, 15, 1176), + (0.05, 18, 9636), + (0.02, 17, 1842), + (0.02, 18, 3791), + (0.01, 18, 72), + ] { + assert!( + classic_failure_probability(precision, eps, largest) <= 0.01, + "eps={eps} p={precision} n={largest}" + ); + assert!( + classic_failure_probability(precision, eps, largest + 1) > 0.01, + "eps={eps} p={precision} n={}", + largest + 1 + ); + } + for too_large in [u64::from(MAX_CONTRACT_DISTINCT) + 1, u64::MAX] { + assert_eq!(classic_failure_probability(18, 0.05, too_large), 1.0); + } +} + +/// Exact occupancy probabilities independently check both tails for every +/// population up to the bound. +#[test] +fn probability_bound_dominates_exact_occupancy_distribution() { + for precision in 4..=10 { + let m = 1usize << precision; + let max_n = 64.min(m / 2); + for eps in [0.05, 0.2, 0.6] { + let model = ClassicHllConfidence::new(max_n as u32, eps).unwrap(); + let Some(bound) = model.failure_probability(precision) else { + continue; + }; + let mut occupancy = vec![0.0; max_n + 1]; + occupancy[0] = 1.0; + for n in 1..=max_n { + let mut next = vec![0.0; max_n + 1]; + for k in 0..n { + next[k] += occupancy[k] * k as f64 / m as f64; + next[k + 1] += occupancy[k] * (m - k) as f64 / m as f64; + } + occupancy = next; + let actual: f64 = occupancy + .iter() + .enumerate() + .filter_map(|(k, &prob)| { + let estimate = -(m as f64) * (-(k as f64) / (m as f64)).ln_1p(); + ((estimate - n as f64).abs() > eps * n as f64).then_some(prob) + }) + .sum(); + assert!( + actual <= bound + 1e-12, + "p={precision} n={n} eps={eps}: {actual}>{bound}" + ); + } + } + } +} + +/// `COUNT(DISTINCT user_id)` within relative 0.05 at δ = 0.01 from classic +/// HLL at precision 18 is certified over a source of at most 9,636 +/// distinct users per evaluation; over 1,000,000 (outside the contract) and +/// without a bound only the relative standard error is stated, whose +/// failure probability is unknown. +#[test] +fn a_distinct_count_is_certified_over_the_source_s_distinct_bound() { + let precision_18 = SketchParams::Hll { precision: 18 }; + let at = |bound| guarantee(&precision_18, 0.01, bound).unwrap(); + let certified = at(Some(9_636)); + assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::Relative); + let error = certified.spec.error.evaluate().unwrap(); + assert!(error <= 0.05 && error > 0.04, "{error}"); + assert!(certified.spec.failure_probability.evaluate().unwrap() <= 0.01); + for uncertified in [Some(1_000_000), None] { + let guarantee = at(uncertified); + assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); + assert_eq!(guarantee.spec.failure_probability.evaluate(), None); + } +} diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 0227d0008..c26786881 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -1,6 +1,7 @@ //! Dispatch committed estimator parameters to their accuracy models. use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr}; use asap_types::ir::schema::FieldDataType; +use asap_types::stages::ValueSign; use asap_types::summary_type::{SketchParams, SketchStatistic}; use asap_types::summary_type::SummaryTypeCapability; @@ -23,21 +24,32 @@ use binomial::binomial_upper_tail; /// `failure_probability`; where structural parameters set them apart (a /// Count-Min's width and depth, HLL's precision), the one θ fixes, whose /// failure probability may exceed `failure_probability` or be unknown -/// (HLL's). A Hydra state reads each subpopulation's state, so it has -/// that state's guarantee. +/// (HLL's, unless the source declares its distinct bound +/// `max_distinct_per_evaluation`: classic HLL's is then certified at +/// `failure_probability`; DDSketch's, unless `state_sign`, the sign of the +/// values the state was updated with, is one sign). A Hydra state reads +/// each subpopulation's state, so it has that state's guarantee. fn guarantee_of_parameters( params: &SketchParams, query: &SketchStatistic, failure_probability: f64, + state_sign: Option, + max_distinct_per_evaluation: Option, ) -> Option { match params { SketchParams::Kll { .. } => kll::guarantee(params, failure_probability), SketchParams::Kmv { .. } => cardinality::guarantee(params, failure_probability), - SketchParams::Hydra(hydra) => { - guarantee_of_parameters(&hydra.per_subpopulation(), query, failure_probability) + SketchParams::Hydra(hydra) => guarantee_of_parameters( + &hydra.per_subpopulation(), + query, + failure_probability, + state_sign, + max_distinct_per_evaluation, + ), + SketchParams::DDSketch { .. } => ddsketch::guarantee(params, state_sign), + SketchParams::Hll { .. } => { + hll::guarantee(params, failure_probability, max_distinct_per_evaluation) } - SketchParams::DDSketch { .. } => ddsketch::guarantee(params), - SketchParams::Hll { .. } => hll::guarantee(params), SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => cms::guarantee(params), SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { count_sketch::guarantee(params) @@ -49,19 +61,33 @@ fn guarantee_of_parameters( /// The guarantee [`theoretical_guarantee`] gives, before it checks that θ /// fails at most `failure_probability` rarely: its metric, and whether its /// failure probability is known at all. `None` when the summary type does -/// not answer `statistic` or has no error model for it. +/// not answer `statistic` or has no error model for it. `state_sign` is +/// the sign of the values the read state was updated with +/// (`GroupProperties::state_column_sign`), which DDSketch's guarantee +/// reads; `max_distinct_per_evaluation` is the source's declared distinct +/// bound (`Memo::max_distinct_per_evaluation`), which a distinct count +/// reads. Without them, the guarantee's metric still holds, with an +/// unknown failure probability. pub(crate) fn guarantee_before_failure_probability_check( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, failure_probability: f64, + state_sign: Option, + max_distinct_per_evaluation: Option, ) -> Option { if !matches!(summary_type, FieldDataType::Sketch(_)) || !SummaryTypeCapability::of(summary_type).answers(statistic) { return None; } - guarantee_of_parameters(parameters, statistic, failure_probability) + guarantee_of_parameters( + parameters, + statistic, + failure_probability, + state_sign, + max_distinct_per_evaluation, + ) } /// The guarantee of an error at most `bound` in `metric`, failing with @@ -89,18 +115,27 @@ fn bounded_guarantee( /// that rarely. `None` when this θ cannot fail that rarely, when the /// summary type does not answer `statistic` (its capability row) or has no /// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch -/// readouts without a certified bound). +/// readouts without a certified bound). A quantile from DDSketch is +/// certified only over values of one sign, `state_sign` +/// (`GroupProperties::state_column_sign`; `accuracy-models.md`, "Sign +/// conditions"), and a distinct count from classic HLL only over a source +/// of at most `max_distinct_per_evaluation` distinct items +/// (`Memo::max_distinct_per_evaluation`); `None` without them. pub fn theoretical_guarantee( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, failure_probability: f64, + state_sign: Option, + max_distinct_per_evaluation: Option, ) -> Option { guarantee_before_failure_probability_check( summary_type, parameters, statistic, failure_probability, + state_sign, + max_distinct_per_evaluation, ) .filter(|guarantee| holds_at_failure_probability(guarantee, failure_probability)) } diff --git a/crates/accuracy/src/guarantee/tests.rs b/crates/accuracy/src/guarantee/tests.rs index f53608525..370487a68 100644 --- a/crates/accuracy/src/guarantee/tests.rs +++ b/crates/accuracy/src/guarantee/tests.rs @@ -2,6 +2,10 @@ use super::*; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::summary_type::SketchAlgorithm; +/// Values of one sign, which every listed guarantee holds over (DDSketch's +/// needs it). +const ONE_SIGN: Option = Some(ValueSign::NonNegative); + /// Every listed θ has a theoretical guarantee for the statistic its /// algorithm is listed for, at failure probability 0.01, in the /// algorithm's metric; along a list of one structure the error never @@ -49,8 +53,15 @@ fn every_listed_parameters_has_a_guarantee_at_one_percent() { let errors: Vec = parameter_list(algorithm) .iter() .map(|parameters| { - let guarantee = theoretical_guarantee(&summary_type, parameters, &statistic, 0.01) - .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); + let guarantee = theoretical_guarantee( + &summary_type, + parameters, + &statistic, + 0.01, + ONE_SIGN, + None, + ) + .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); assert_eq!(guarantee.spec.accuracy_metric, metric, "{algorithm:?}"); guarantee.spec.error.evaluate().unwrap() }) @@ -77,7 +88,7 @@ fn a_guarantee_is_evaluated_at_the_required_failure_probability() { let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); let k = SketchParams::Kll { k: 547 }; let error_at = |delta| { - theoretical_guarantee(&kll, &k, &quantile, delta) + theoretical_guarantee(&kll, &k, &quantile, delta, None, None) .unwrap() .spec .error @@ -92,9 +103,37 @@ fn a_guarantee_is_evaluated_at_the_required_failure_probability() { value: None, }; let depth = |depth| SketchParams::Cms { width: 272, depth }; - assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2).is_some()); - assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4).is_none()); - assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2, None, None).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4, None, None).is_none()); + assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4, None, None).is_some()); // A statistic the summary type does not answer has no guarantee. - assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01).is_none()); + assert!( + theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01, None, None).is_none() + ); +} + +/// DDSketch's relative error on a quantile holds only over values of one +/// sign: over non-negative or non-positive values at α, at any failure +/// probability; over values of unknown sign, or of no known sign, the +/// guarantee is not certified at any failure probability, though its +/// metric is still relative error. +#[test] +fn a_ddsketch_quantile_is_certified_only_over_values_of_one_sign() { + let ddsketch = FieldDataType::Sketch(SketchAlgorithm::DDSketch); + let alpha = SketchParams::DDSketch { alpha: 0.01 }; + let quantile = SketchStatistic::Quantile { q: 0.99 }; + for (sign, certified) in [ + (Some(ValueSign::NonNegative), true), + (Some(ValueSign::NonPositive), true), + (Some(ValueSign::Unknown), false), + (None, false), + ] { + let theoretical = theoretical_guarantee(&ddsketch, &alpha, &quantile, 0.01, sign, None); + assert_eq!(theoretical.is_some(), certified, "{sign:?}"); + let before_check = guarantee_before_failure_probability_check( + &ddsketch, &alpha, &quantile, 0.01, sign, None, + ) + .unwrap(); + assert_eq!(before_check.spec.accuracy_metric, AccuracyMetric::Relative); + } } diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index abf154e7d..3b049a6f9 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -5,6 +5,7 @@ //! ([`AccuracyEvidence`], in `evidence`), each operator's error contract //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the +//! metrics each query root allows, [`query_roots_with_target`], and the //! metric of a readout's error from `readout`), each readout's required //! accuracy ([`readout_levels`], in `readout_caps`), and $\alpha^{\log}$ of //! a logical edge ([`ErrorMetricsOfMemo::edge_meets_logical_requirements`], @@ -26,6 +27,6 @@ mod readout_caps; pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; -pub use logical_edge::within_limit; -pub use metric_rule::ErrorMetricsOfMemo; +pub use logical_edge::{tolerated_limit, within_limit}; +pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/logical_edge/mod.rs b/crates/accuracy/src/logical_edge/mod.rs index 56542b462..e72cdb8e3 100644 --- a/crates/accuracy/src/logical_edge/mod.rs +++ b/crates/accuracy/src/logical_edge/mod.rs @@ -21,7 +21,16 @@ const SATISFACTION_TOLERANCE: f64 = 1e-9; /// `limit`: at most `limit` widened by [`SATISFACTION_TOLERANCE`], so a /// bound equal to it up to floating-point noise meets it. pub fn within_limit(evaluated: f64, limit: f64) -> bool { - evaluated <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON + evaluated <= tolerated_limit(limit) +} + +/// The largest value [`within_limit`] admits for `limit`: `limit` widened +/// by [`SATISFACTION_TOLERANCE`]. Stage 3's root checks of constraint (K) +/// bound a composed error by it, so they accept what an edge accepts. +/// +/// Example: a limit of 0.01 admits up to 0.01 · (1 + 1e-9) + 2.2e-16. +pub fn tolerated_limit(limit: f64) -> f64 { + limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON } impl ErrorMetricsOfMemo { @@ -37,7 +46,11 @@ impl ErrorMetricsOfMemo { /// guarantee of the producer's θ for the readout's statistic, at the /// required failure probability, is within the required error /// ([`within_limit`]) and in its metric: the deployment's measurement - /// (`evidence`) when it has one, else the theoretical guarantee; a + /// (`evidence`) when it has one, else the theoretical guarantee over + /// the sign of the state's values (`GroupProperties::state_column_sign`, + /// DDSketch's sign condition) and, for a distinct count, the scanned + /// sources' declared distinct bound + /// (`Memo::max_distinct_per_evaluation`); a /// merged state (any producer but a build) must keep the guarantee /// (`MergedError::Exact` or `AsIfBuiltOverUnion`). A top-k's heap /// holds its `k` whatever the requirement. @@ -68,7 +81,8 @@ impl ErrorMetricsOfMemo { }) } }; - let Operator::ASAP(ASAPOp::SummaryEstimate { .. }) = consumer_candidate.expression.operator + let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = + consumer_candidate.expression.operator else { let Some(producer_metric) = self.readout_error_metric(producer) else { return Ok(()); @@ -102,7 +116,8 @@ impl ErrorMetricsOfMemo { evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); let allowed = self.error_metrics_allowed_on_input_edge(consumer, input); // The theoretical guarantee before its failure-probability check, - // computed once for both checks below. + // computed once for both checks below, over the sign of the values + // the read states were updated with. let theoretical = (measured.is_none() && (allowed.is_some() || required.is_some())) .then(|| { guarantee_before_failure_probability_check( @@ -110,6 +125,10 @@ impl ErrorMetricsOfMemo { parameters, statistic, failure_probability, + memo.groups()[summary_input.0 as usize] + .group_properties + .state_column_sign(), + memo.max_distinct_per_evaluation(summary_input), ) }) .flatten(); diff --git a/crates/accuracy/src/logical_edge/tests.rs b/crates/accuracy/src/logical_edge/tests.rs index 40d51baf9..7f31c343b 100644 --- a/crates/accuracy/src/logical_edge/tests.rs +++ b/crates/accuracy/src/logical_edge/tests.rs @@ -4,7 +4,10 @@ use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; -use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; +use asap_types::stages::{ + CandidateForMemoGroup, LogicalRequirements, OriginOfCandidateForMemoGroup, RequiredAccuracy, + ValueSign, +}; use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_builders::query; use asap_types::test_support::memo_fixtures::scan; @@ -145,3 +148,136 @@ fn a_heap_smaller_than_k_fails_the_topk_readouts_edge() { } assert_eq!(small, 2 * 28); } + +/// The edge from the build of `algorithm` at `parameters` over `Scan events` (whose +/// facts are `facts`) into a readout of `statistic` required at +/// `required`, which a query with `target` reads. +fn readout_edge( + facts: asap_types::workload::SourceFacts, + algorithm: SketchAlgorithm, + parameters: SketchParams, + statistic: SketchStatistic, + required: RequiredAccuracy, + target: AccuracyTarget, +) -> Result<(), RootPruneReason> { + let data = DataWorkload { + sources: [("events".to_string(), facts)].into(), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + let events = memo.find_or_add_group(scan("events", &[], vec![])).unwrap(); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: events, + summary_type: FieldDataType::Sketch(algorithm), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let state = memo.find_or_add_group(build.clone()).unwrap(); + let producer = memo + .add_candidate( + state, + CandidateForMemoGroup::new( + build, + Some(parameters), + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) + .unwrap(); + let readout = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: statistic, + }); + let group = memo.find_or_add_group(readout.clone()).unwrap(); + let mut candidate = CandidateForMemoGroup::new( + readout, + None, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ); + candidate.input_requirements = vec![LogicalRequirements { + required_accuracy: Some(required), + }]; + let consumer = memo.add_candidate(group, candidate).unwrap(); + query(&mut memo, group, Some(target)); + let order = memo.topological_order().expect("acyclic"); + ErrorMetricsOfMemo::of(&memo, &order).edge_meets_logical_requirements( + &memo, + producer, + consumer, + 0, + &NoMeasuredGuarantees, + ) +} + +/// `COUNT(DISTINCT user_id)` within relative 0.05 at δ = 0.01 from classic +/// HLL at precision 18: the edge holds over a source declaring at most +/// 9,636 distinct items per evaluation, and fails with `NoGuarantee` over +/// 1,000,000 (outside the contract) or without a bound. +#[test] +fn a_distinct_count_readout_reads_the_scanned_sources_distinct_bound() { + for (bound, holds) in [(Some(9_636), true), (Some(1_000_000), false), (None, false)] { + let facts = asap_types::workload::SourceFacts { + max_distinct_per_evaluation: bound, + ..Default::default() + }; + let edge = readout_edge( + facts, + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 18 }, + SketchStatistic::Cardinality, + RequiredAccuracy { + metric: AccuracyMetric::Relative, + max_error: 0.05, + failure_probability: 0.01, + }, + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ); + match holds { + true => assert!(edge.is_ok(), "{bound:?}: {edge:?}"), + false => assert!( + matches!(edge, Err(RootPruneReason::NoGuarantee { input: 0, .. })), + "{bound:?}: {edge:?}" + ), + } + } +} + +/// DDSketch's relative error on a quantile holds for values of one sign: +/// `quantile_over_time(0.5, events[..])` at relative 0.01 from α = 0.01 +/// holds when `events`' value is declared non-negative, and fails with +/// `NoGuarantee` when its sign is unknown. +#[test] +fn a_ddsketch_quantile_holds_only_over_values_of_one_sign() { + for (sign, holds) in [(Some(ValueSign::NonNegative), true), (None, false)] { + let facts = asap_types::workload::SourceFacts { + column_signs: sign + .map(|sign| ("value".to_string(), sign)) + .into_iter() + .collect(), + ..Default::default() + }; + let edge = readout_edge( + facts, + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + SketchStatistic::Quantile { q: 0.5 }, + RequiredAccuracy { + metric: AccuracyMetric::Relative, + max_error: 0.01, + failure_probability: 1.0, + }, + AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative), + ); + match holds { + true => assert!(edge.is_ok(), "{sign:?}: {edge:?}"), + false => assert!( + matches!(edge, Err(RootPruneReason::NoGuarantee { input: 0, .. })), + "{sign:?}: {edge:?}" + ), + } + } +} diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 9d15c30be..6e947e0e4 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -15,12 +15,9 @@ use crate::error_contract::{error_contract, ErrorContract}; use crate::readout::sketch_readout_error_metric; use crate::readout_caps::caps; -/// How many error metrics there are: `AccuracyMetric::ALL`'s length. -pub(crate) const METRIC_COUNT: usize = AccuracyMetric::ALL.len(); - /// Per metric, in `AccuracyMetric::ALL` order: a candidate's error /// contract in it; `None` = no contract. -pub(crate) type ErrorContractPerMetric = [Option; METRIC_COUNT]; +pub(crate) type ErrorContractPerMetric = [Option; AccuracyMetric::COUNT]; /// The error metrics of one memo state, built once per memo state before /// an edge pass; Stage 3 reads the same contracts. @@ -103,9 +100,8 @@ impl ErrorMetricsOfMemo { candidate: CandidateForMemoGroupId, metric: AccuracyMetric, ) -> Option<&ErrorContract> { - self.error_contract[candidate.group.0 as usize][candidate.index_in_group] - [metric_index(metric)] - .as_ref() + self.error_contract[candidate.group.0 as usize][candidate.index_in_group][metric.index()] + .as_ref() } /// The error metric of `candidate` when it is a sketch readout. @@ -272,20 +268,14 @@ impl ErrorMetricsOfMemo { } } -/// The index of `metric` in `AccuracyMetric::ALL`. -fn metric_index(metric: AccuracyMetric) -> usize { - AccuracyMetric::ALL - .iter() - .position(|&listed| listed == metric) - .expect("AccuracyMetric::ALL lists every metric") -} - /// Each query with a target, with the groups its root reads and the metrics /// each allows: an operator root, `scalar(v)` and an SQL scalar subquery /// (which return the value they read) allow the target's metrics /// (`AccuracyTarget::metrics`); any other scalar expression's operands /// allow none, as no error contract covers it, so they are read exactly. -pub(crate) fn query_roots_with_target( +/// Pass 2's readout caps and Stage 3's root checks of constraint (K) read +/// it. +pub fn query_roots_with_target( memo: &Memo, ) -> Vec<(usize, MemoGroupId, &AccuracyTarget, Vec)> { let mut roots = Vec::new(); diff --git a/crates/accuracy/src/readout/mod.rs b/crates/accuracy/src/readout/mod.rs index fc4c9b8bc..7e599e082 100644 --- a/crates/accuracy/src/readout/mod.rs +++ b/crates/accuracy/src/readout/mod.rs @@ -24,9 +24,16 @@ pub(crate) fn readout_error_metric( }; let parameters = parameter_list(*algorithm).first()?; Some( - guarantee_before_failure_probability_check(summary_type, parameters, statistic, 1.0)? - .spec - .accuracy_metric, + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + 1.0, + None, + None, + )? + .spec + .accuracy_metric, ) } diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index d4bb4c854..3d3016bf3 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -169,10 +169,10 @@ fn group_levels( /// depend on does, so by the union bound δ splits over all of them, /// `readout_values` ([`ReadoutsReadByQueries::readout_values`], summed over /// every readout group below the query), and over the values its scope -/// covers, $V$ (`AccuracyScope::max_values_covered`, $n_q$ for a whole -/// result): the per-value failure probability is -/// $\delta / \max(V, \text{readout values})$, and needs both known when the -/// target sets a δ. +/// covers, $V$ (`AccuracyTarget::failure_probability_per_output_value`, +/// $\delta / V$ with $V = n_q$ for a whole result): the per-value failure +/// probability is $\min(\delta / V, \delta / \text{readout values})$, and +/// needs both known when the target sets a δ. /// - `max(quantile_over_time(0.9, m[5m]))` over 1000 series at δ = 0.01 /// returns one value but reads 1000 readout values: 1e-5. /// - `max by (job) (q{host="h0"} or q{host="h1"})` with 3 series per side @@ -192,12 +192,9 @@ fn query_levels( ) -> Option> { let per_value_failure_probability = match target.failure_probability { None => 1.0, - Some(delta) => { - let values_covered = target - .accuracy_scope - .max_values_covered(number_of_values_per_query_evaluation)?; - delta / values_covered.max(readout_values?).max(1) as f64 - } + Some(delta) => target + .failure_probability_per_output_value(number_of_values_per_query_evaluation)? + .min(delta / readout_values?.max(1) as f64), }; let level = |max_error, failure_probability| RequiredAccuracy { metric, diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 7b1b04399..26f6a3108 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -63,7 +63,9 @@ // `PhysicalASAPDAG`) and its `Selection`: `accepted_accuracy_per_query`, // `pruned_candidates`, `milp_objective_cost_per_second` (the selected // plan's cost per second), `milp_variables`, `milp_constraints`, -// `milp_construction_ms` and `milp_solve_ms`; `error` when no plan was +// `milp_construction_ms`, `milp_solve_ms` and `milp_solve_count` (2 when +// the solve at HiGHS's default tolerances missed an accuracy target and +// was repeated at 1e-10); `error` when no plan was // selected. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them; `materialized_views_error` when the @@ -644,6 +646,7 @@ fn selection_json(selection: &Result<(PhysicalASAPDAG, Selection), SelectionErro "milp_constraints": selection.milp_constraint_count, "milp_construction_ms": selection.milp_construction_ms, "milp_solve_ms": selection.milp_solve_ms, + "milp_solve_count": selection.milp_solve_count, }) } diff --git a/crates/integration-tests/tests/accuracy_at_query_roots.rs b/crates/integration-tests/tests/accuracy_at_query_roots.rs new file mode 100644 index 000000000..3ef77fac5 --- /dev/null +++ b/crates/integration-tests/tests/accuracy_at_query_roots.rs @@ -0,0 +1,207 @@ +//! Constraint (K), each query's composed accuracy at its root, through +//! every planning stage on a two-query example (`accuracy-models.md`, +//! "Example: allocation in a ratio"): A, the p95 of `latency` at rank +//! error 0.01, and B, the ratio of its p99 and p50 at relative error 0.03. + +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; +use asap_planner::{DeploymentCapabilities, PlanningModels}; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::summary_type::SketchParams; +use asap_types::workload::PlanningWorkload; + +/// PromQL `queries`, each `(query, accuracy JSON)` every 10 s, over +/// `latency` (1,000 series, one sample per second, ingested, values +/// declared non-negative). +fn workload(queries: &[(&str, &str)]) -> PlanningWorkload { + let entries: Vec = queries + .iter() + .map(|(query, accuracy)| { + format!( + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {accuracy}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}"# + ) + }) + .collect(); + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, + "sources": {{"latency": {{"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000, + "column_signs": {{"value": "non_negative"}}}}}}}}}}"#, + entries.join(",") + )) + .unwrap() +} + +/// The plan Stage 3 selects for `queries`, no raw data kept (every query +/// reads summaries built at ingestion), with accuracy enforced or +/// ignored. +fn plan(queries: &[(&str, &str)], ignore_accuracy: bool) -> PhysicalASAPDAG { + selection(queries, ignore_accuracy).0 +} + +/// [`plan`], with the accepted accuracy Stage 3 reports per query. +fn selection(queries: &[(&str, &str)], ignore_accuracy: bool) -> (PhysicalASAPDAG, Vec) { + let workload = workload(queries); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap(); + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..Default::default() + }; + let models = PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_ignore_accuracy_guarantees(ignore_accuracy); + let roots = lower_promql_roots(&workload).unwrap(); + let run = run_planning_stages_over(roots, &entries, &data, models).unwrap(); + let (plan, selection) = run.run.selection.as_ref().expect("a plan is selected"); + let accepted = selection + .accepted_accuracy_per_query + .iter() + .map(ToString::to_string) + .collect(); + (plan.clone(), accepted) +} + +/// Queries A and B of the module's example. +const PROBLEM_EXAMPLE: [(&str, &str); 2] = [ + ( + "quantile_over_time(0.95, latency[1h])", + r#"{"accuracy_metric": "rank", "max_error": 0.01}"#, + ), + ( + "quantile_over_time(0.99, latency[1h]) / quantile_over_time(0.5, latency[1h])", + r#"{"accuracy_metric": "relative", "max_error": 0.03}"#, + ), +]; + +/// The θ of the plan's sketch state nodes. +fn sketch_parameters(plan: &PhysicalASAPDAG) -> Vec { + plan.nodes + .iter() + .filter_map(|node| node.parameters.clone()) + .collect() +} + +/// A meets rank error 0.01 only with a KLL of k ≥ 269 (a DDSketch readout +/// bounds a relative error, not a rank); B reads DDSketch states whose +/// readouts' composed bound $(\alpha_{p99} + \alpha_{p50}) / (1 - 0.1)$ is +/// within 0.03, which two α = 0.02 readouts (each within 0.03 alone) are +/// not. +#[test] +fn each_query_meets_its_target_at_its_root() { + let (plan, accepted) = selection(&PROBLEM_EXAMPLE, false); + assert_eq!( + accepted, + ["accepted rank (Kll)", "accepted relative (DDSketch)"], + "the metric of each query's readouts" + ); + let parameters = sketch_parameters(&plan); + assert!( + parameters + .iter() + .any(|p| matches!(p, SketchParams::Kll { k } if *k >= 269)), + "A reads a KLL meeting rank 0.01: {parameters:?}" + ); + assert!( + !parameters + .iter() + .any(|p| matches!(p, SketchParams::Kll { k } if *k < 269)), + "{parameters:?}" + ); + let mut alphas: Vec = parameters + .iter() + .filter_map(|p| match p { + SketchParams::DDSketch { alpha } => Some(*alpha), + _ => None, + }) + .collect(); + alphas.sort_by(f64::total_cmp); + let (Some(&smallest), Some(&largest)) = (alphas.first(), alphas.last()) else { + panic!("B reads DDSketch: {parameters:?}"); + }; + assert!( + (smallest + largest) / (1.0 - 0.1) <= 0.03 * (1.0 + 1e-9), + "B's composed bound: {alphas:?}" + ); +} + +/// Ignoring accuracy, no query has a target: (K) has no rows and any +/// readout serves, so one summary type answers A's p95 and B's p99 and p50 +/// (which one is a matter of cost), where enforcing accuracy needs a KLL +/// for A's rank target and DDSketch for B's relative one. The report says +/// there is no target to accept (it said "accepted exact (…)"). +#[test] +fn ignoring_accuracy_plans_without_k() { + // The number of sketch algorithms the plan builds. + let algorithms = |plan: &PhysicalASAPDAG| { + sketch_parameters(plan) + .iter() + .map(std::mem::discriminant) + .collect::>() + .len() + }; + assert_eq!(algorithms(&plan(&PROBLEM_EXAMPLE, false)), 2); + let (ignored, accepted) = selection(&PROBLEM_EXAMPLE, true); + assert_eq!(algorithms(&ignored), 1, "{:?}", sketch_parameters(&ignored)); + assert!( + accepted + .iter() + .all(|accepted| accepted.starts_with("no accuracy target (")), + "{accepted:?}" + ); +} + +/// A whole-result failure probability of 0.01 over 1,000 series is 1e-5 +/// per value: a KLL read at that failure probability meets rank error +/// 0.01 only from k = 547 up (`accuracy-models.md`, the KLL row of the +/// built-in models: 2.296 / k^0.9723 scaled by the Gaussian quantile ratio). +#[test] +fn a_whole_result_failure_probability_is_divided_over_the_values() { + let plan = plan( + &[( + "quantile_over_time(0.95, latency[1h])", + r#"{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}"#, + )], + false, + ); + let parameters = sketch_parameters(&plan); + assert!(!parameters.is_empty()); + for parameters in parameters { + assert!( + matches!(parameters, SketchParams::Kll { k } if k >= 547), + "{parameters:?}" + ); + } +} + +/// p90, p50 and p90 / p50 over `latency[15m]`, each at relative 0.05: the +/// two quantiles alone accept DDSketch readouts at α = 0.05, and sharing +/// those with the ratio is cheapest, but the ratio's composed bound would be +/// (0.05 + 0.05) / (1 − 0.1) = 0.111. (K) is what rules that out: the plan +/// reads readouts at α ≤ 0.02 for the ratio, (0.02 + 0.02) / 0.9 = 0.044. +#[test] +fn a_ratio_sharing_its_readouts_with_its_operands_still_meets_its_target() { + let target = r#"{"accuracy_metric": "relative", "max_error": 0.05}"#; + let plan = plan( + &[ + ("quantile_over_time(0.9, latency[15m])", target), + ("quantile_over_time(0.5, latency[15m])", target), + ( + "quantile_over_time(0.9, latency[15m]) / quantile_over_time(0.5, latency[15m])", + target, + ), + ], + false, + ); + let alphas: Vec = sketch_parameters(&plan) + .iter() + .filter_map(|p| match p { + SketchParams::DDSketch { alpha } => Some(*alpha), + _ => None, + }) + .collect(); + assert!( + alphas.iter().any(|&alpha| alpha <= 0.02), + "the ratio reads readouts composing within 0.05: {alphas:?}" + ); +} diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index 9ee627c25..b650d32e6 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -6,6 +6,7 @@ mod physical_common; mod planner_layering_common; mod planning_common; +mod accuracy_at_query_roots; mod aggregate; mod binary_op; mod crate_graph; diff --git a/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs b/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs index 77f0afff8..c84f116b1 100644 --- a/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs +++ b/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs @@ -47,7 +47,8 @@ fn live_sketch_readouts(logical: &LogicalASAPMemoCandidates) -> usize { /// `promql_drop_metric_name`. Its contract judges only the value: `abs` /// and the negation keep an absolute error, scaling by a constant a /// relative one, so a sketch readout below each still serves the query, as -/// it does for the bare readout. +/// it does for the bare readout (DDSketch's quantile over `latency`, whose +/// values are declared non-negative). #[test] fn a_function_of_the_value_keeps_the_readouts_below_it() { let absolute = json!({"max_error": 0.05, "accuracy_metric": "absolute"}); @@ -56,8 +57,8 @@ fn a_function_of_the_value_keeps_the_readouts_below_it() { ("count_over_time(m[1m])", &absolute), ("abs(count_over_time(m[1m]))", &absolute), ("-count_over_time(m[1m])", &absolute), - ("quantile_over_time(0.9, m[5m]) / 2", &relative), - ("quantile_over_time(0.9, m[5m]) * 2", &relative), + ("quantile_over_time(0.9, latency[5m]) / 2", &relative), + ("quantile_over_time(0.9, latency[5m]) * 2", &relative), ] { let logical = pass2(query, accuracy.clone()); assert!(live_sketch_readouts(&logical) > 0, "{query}"); diff --git a/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs b/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs index 127f8687d..fb3c67dfa 100644 --- a/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs +++ b/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs @@ -44,14 +44,14 @@ fn live(logical: &LogicalASAPMemoCandidates, group: MemoGroupId) -> bool { }) } -/// `quantile_over_time(0.9, m[5m])` twice, Q0 at rank error 0.01 and Q1 at -/// relative error 0.05: both share the quantile group and both the KLL -/// (rank) and DDSketch (relative) readouts survive, but Q0 reads only the -/// KLL readout and Q1 only the DDSketch one; each reads the other's -/// readout groups only exactly. +/// `quantile_over_time(0.9, latency[5m])` twice (values non-negative), Q0 +/// at rank error 0.01 and Q1 at relative error 0.05: both share the +/// quantile group and both the KLL (rank) and DDSketch (relative) readouts +/// survive, but Q0 reads only the KLL readout and Q1 only the DDSketch +/// one; each reads the other's readout groups only exactly. #[test] fn each_query_reads_only_the_readouts_in_its_targets_metric() { - let query = "quantile_over_time(0.9, m[5m])"; + let query = "quantile_over_time(0.9, latency[5m])"; let logical = promql_pass2(&[ (query, json!({"max_error": 0.01, "accuracy_metric": "rank"})), ( @@ -84,7 +84,7 @@ fn each_query_reads_only_the_readouts_in_its_targets_metric() { } } -/// `count(quantile_over_time(0.9, m[5m]))` at relative error 0.05: the +/// `count(quantile_over_time(0.9, latency[5m]))` at relative error 0.05: the /// count reads the quantiles through an exact count, whose contract /// passes every metric, while Pass 1's Count-Min and UnivMon builds over /// the quantiles (no contract) are dead. A dead candidate makes no group @@ -92,7 +92,7 @@ fn each_query_reads_only_the_readouts_in_its_targets_metric() { #[test] fn a_dead_candidate_makes_no_group_exact_only() { let logical = promql_pass2(&[( - "count(quantile_over_time(0.9, m[5m]))", + "count(quantile_over_time(0.9, latency[5m]))", json!({"max_error": 0.05, "failure_probability": 0.01, "accuracy_metric": "relative"}), )]); let memo = &logical.memo; diff --git a/crates/integration-tests/tests/planning_common/mod.rs b/crates/integration-tests/tests/planning_common/mod.rs index fe0ae4df0..e962f2d52 100644 --- a/crates/integration-tests/tests/planning_common/mod.rs +++ b/crates/integration-tests/tests/planning_common/mod.rs @@ -85,8 +85,9 @@ pub fn selected_plan(w: &FuzzWorkload, settings: &CostInputs) -> PhysicalASAPDAG } /// Pass 2's output for PromQL `queries`, each with its accuracy target, -/// repeating every minute over the gauge `m` and the counter `c`, 1000 -/// series each. +/// repeating every minute over the gauge `m`, the counter `c` and the +/// gauge `latency` whose values are declared non-negative (so DDSketch's +/// quantile guarantee holds over it), 1000 series each. pub fn promql_pass2(queries: &[(&str, Value)]) -> LogicalASAPMemoCandidates { pass2_of( json!("promql"), @@ -94,6 +95,8 @@ pub fn promql_pass2(queries: &[(&str, Value)]) -> LogicalASAPMemoCandidates { json!({ "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + "latency": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000, + "column_signs": {"value": "non_negative"}}, }), None, ) diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs index 158e83b7c..3a24e3c29 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs @@ -8,7 +8,6 @@ use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; use asap_types::summary_type::parameter_list; use asap_types::summary_type::{SketchAlgorithm, SketchParams}; -use asap_types::workload::DataWorkload; use crate::test_fixtures::*; use asap_types::test_support::memo_builders::*; @@ -21,10 +20,11 @@ use asap_types::test_support::memo_fixtures::{arithmetic, range}; /// pruned by the metric rule and every KLL build through it. The readouts /// of DDSketch get the levels 0.02 … 0.001 (cap 0.045), so the builds with /// alpha 0.1 and 0.05 miss every level they could serve. Chain groups are -/// added only for the surviving builds. +/// added only for the surviving builds. `m`'s values are declared +/// non-negative, so DDSketch's quantile guarantee holds. #[test] fn dead_candidates_are_pruned_and_listed_before_chain_groups() { - let mut memo = Memo::new(&DataWorkload::default()); + let mut memo = Memo::new(&with_non_negative_values("m")); let input = scan_group(&mut memo, "m", &["job"], vec![]); let window = group(&mut memo, range(300_000, input)); let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs index 4e9827a4a..a4cbbe366 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs @@ -5,7 +5,6 @@ use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; use asap_types::stages::{ColumnLineage, Memo}; use asap_types::summary_type::SketchAlgorithm; -use asap_types::workload::DataWorkload; use crate::test_fixtures::*; use asap_types::test_support::memo_builders::*; @@ -18,9 +17,11 @@ use asap_types::test_support::memo_fixtures::{arithmetic, range}; /// division carries relative error); Q1's comparison has no contract, so /// Q1 reads the ratio and every group below it only exactly. (Both read /// the rows below a sketch build exactly: a build has no contract.) +/// `latency` is declared non-negative, so DDSketch's quantile guarantee +/// holds. #[test] fn a_query_comparing_a_shared_ratio_reads_it_only_exactly() { - let mut memo = Memo::new(&DataWorkload::default()); + let mut memo = Memo::new(&with_non_negative_values("latency")); let input = scan_group(&mut memo, "latency", &["job"], vec![]); let window = group(&mut memo, range(300_000, input)); let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); diff --git a/crates/stage2-physical-optimization/src/candidate.rs b/crates/stage2-physical-optimization/src/candidate.rs index 86d658ebc..d58b0f928 100644 --- a/crates/stage2-physical-optimization/src/candidate.rs +++ b/crates/stage2-physical-optimization/src/candidate.rs @@ -270,12 +270,8 @@ fn count_min_weights_not_proven_non_negative( ) { return false; } - let properties = &memo.groups()[group.0 as usize].group_properties; - let weight_sign = properties - .schema - .fields - .iter() - .position(|field| !field.is_plain()) - .map(|state| properties.column_signs[state]); + let weight_sign = memo.groups()[group.0 as usize] + .group_properties + .state_column_sign(); weight_sign != Some(ValueSign::NonNegative) } diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 96d153d83..d0ab792a1 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -17,9 +17,12 @@ //! exact candidates only, $a_{q,e}$ marking what its selected path reads. It //! builds the plan (`PhysicalASAPDAG`) from the selected executions and //! edges as its last step ([`plan`], each node retaining its candidate's -//! $h_v$, the needs from [`retention`]). Accuracy at each query root, -//! constraint (K), with per-candidate error bounds $E_c$ and $D_c$ -//! (section 4), is the remaining extension. Costs come from `asap-cost`. +//! $h_v$, the needs from [`retention`]). Accuracy at each query root is +//! constraint (K) (`accuracy-models.md`, "Checking each query's target in +//! selection"): error bounds $E_{\kappa,c,m}$ per execution +//! and metric and failure probabilities $D_{\kappa,c}$, composed along the +//! selected edges and checked at each query's root. Costs come from +//! `asap-cost`. mod cost; mod milp; @@ -116,6 +119,13 @@ pub fn select_plan( .map_err(|error| match error { SelectionMilpError::Infeasible => infeasible(&costs), SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), + SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value, + } => ErrorSelectingPlan::AccuracyTargetBelowSolverPrecision { + query: query.0, + failure_probability_per_output_value, + }, })?; let plan = plan_of(physical, &solved, &outputs).map_err(|error| { ErrorSelectingPlan::Solver(format!("the selected plan is invalid: {error}")) @@ -143,6 +153,7 @@ pub fn select_plan( milp_constraint_count: solved.constraint_count, milp_construction_ms: solved.construction_ms, milp_solve_ms: solved.solve_ms, + milp_solve_count: solved.solve_count, }; Ok((plan, selection)) } diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index 43a76e733..5f530e122 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -1,41 +1,783 @@ //! The accuracy extension of the selection MILP -//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 4): its variables and constraint (K), `todo!()` until (K) is -//! added. +//! (`docs/design_docs/concepts/accuracy-models.md`, "Checking each query's +//! target in selection: constraint (K)"): constraint (K), per execution and +//! per error metric, over the error each execution adds and passes on +//! ([`AccuracyRules`], [`ErrorModel`]), and the same rule checked on one +//! selection ([`AccuracyRules::selection_meets_targets`]), which the greedy +//! upper bound needs to be a feasible plan of (K) and which checks the +//! solver's selection exactly. One composition rule, +//! [`AccuracyRules::compose`], gives both that check and the ceilings of +//! (K)'s big-M rows; [`accuracy_constraints`] writes the same rule as rows. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; +use std::hash::{Hash, Hasher}; -use good_lp::{Constraint, Variable}; +use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; +use asap_accuracy::{query_roots_with_target, tolerated_limit, ErrorContract, ErrorMetricsOfMemo}; use asap_types::accuracy::AccuracyMetric; -use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId}; - -use super::model::SelectionVariables; - -/// The accuracy variables of constraint (K) (section 4), all continuous. -#[allow(dead_code)] // Read once constraint (K) is added. -struct AccuracyVariables { - /// $E_{c,m}$: candidate $c$'s error bound in metric $m$. - error_bound: HashMap<(PhysicalCandidateId, AccuracyMetric), Variable>, - /// $F_{c,i,m}$: the chosen producer's error bound in metric $m$ on input - /// $i$ of $c$. - error_bound_of_input: HashMap<(PhysicalCandidateId, usize, AccuracyMetric), Variable>, - /// $D_c$, by [`PhysicalCandidateId`]. - failure_probability: Vec, - /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. - failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, -} - -/// (K): $E_{c,m}$ per candidate and metric and $D_c$ per candidate, big-M on -/// selected edges, the metric rule on the selected path, and the root -/// checks $E_{v,m_q} \le \varepsilon_q + M_q(1 - z_{q,v})$, -/// $D_v \le \delta_q + (1 - z_{q,v})$. -#[allow(dead_code)] // Called once constraint (K) is added. -fn accuracy_constraints( - physical: &PhysicalASAPMemoCandidates, - selection: &SelectionVariables, +use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId}; + +use super::equivalent_contexts::SharedVariables; +use super::model::SelectionInputs; +use crate::selectable::QueryId; + +/// An execution's error as (K) composes it: its error bound per metric +/// (by [`AccuracyMetric::index`]) and its failure probability. +/// +/// Example: a DDSketch readout required at relative 0.01 with δ = 1e-3 +/// is 0.01 in `Relative`, 0 in every other metric, and 1e-3. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub(super) struct ExecutionError { + /// $E_{c,m}$ per metric. + error_bound_per_metric: [f64; AccuracyMetric::COUNT], + /// $D_c$. + failure_probability: f64, +} + +impl ExecutionError { + /// The larger of `self` and `other` in each metric and in failure + /// probability, as the most any one of several producers passes on. + fn largest(self, other: Self) -> Self { + let mut largest = self; + for (error, other) in largest + .error_bound_per_metric + .iter_mut() + .zip(other.error_bound_per_metric) + { + *error = error.max(other); + } + largest.failure_probability = largest.failure_probability.max(other.failure_probability); + largest + } + + /// Whether it is 0 in every metric and in failure probability. + fn is_zero(&self) -> bool { + self.failure_probability == 0.0 && self.error_bound_per_metric.iter().all(|&e| e == 0.0) + } +} + +/// What composing one execution's error from what its inputs read gives +/// ([`AccuracyRules::compose`]). +struct Composition { + /// Its own error plus each input's error times the contract's factor, + /// and its own failure probability plus each input's. + error: ExecutionError, + /// Whether every input's error passes the execution's metric rule (it + /// has a contract in that metric) and the contract's validity cap. + inputs_admitted: bool, +} + +/// What (K) reads of the memo, built once per selection when some query +/// has an accuracy target: each candidate's error contracts, each +/// execution's own error, which executions a query with a target reaches, +/// and what each query root allows. +pub(super) struct AccuracyRules { + /// The candidates' error contracts per metric + /// (`ErrorMetricsOfMemo::error_contract`). + error_metrics: ErrorMetricsOfMemo, + /// Per execution, by position in `SelectableExecutions::executions`, + /// its logical candidate. + candidate_of_execution: Vec, + /// Per execution, the error it adds: a readout's requirement + /// (`CandidateForMemoGroup::required_accuracy`), 0 for any other. + own_error_per_execution: Vec, + /// Per execution, whether a query with an accuracy target reaches its + /// group. + reached_by_query_with_target: Vec, + /// Every execution, its producers' groups before its own (the memo's + /// topological order): an order that composes each producer before + /// its consumers. + executions_producers_first: Vec, + /// Per (query, output group its root reads), what its root check + /// bounds the output by. + roots: HashMap<(QueryId, MemoGroupId), RootLimits>, +} + +/// What a root check of (K) bounds one query output by, each limit +/// already widened by [`tolerated_limit`] as the edge check widens it. +/// Two outputs with equal limits have the same root rows, so they compare +/// and hash by the limits' bits. +/// +/// Example: `p99 / p50` at relative 0.03 with δ = 0.01 over a whole result +/// of 100 values allows 0.03 (widened) in `Relative`, 0 in every other +/// metric, and 1e-4 (widened) failure probability per output value. +#[derive(Debug, Clone, Copy)] +pub(super) struct RootLimits { + /// Per metric (by [`AccuracyMetric::index`]), the error the output may + /// have: the target's ε in a metric the root allows + /// ([`query_roots_with_target`]), 0 in any other. + tolerated_error_per_metric: [f64; AccuracyMetric::COUNT], + /// The failure probability each output value may have + /// (`AccuracyTarget::failure_probability_per_output_value`); `None` + /// without a δ, or with one over the whole result and $n_q$ unknown + /// (Pass 2 then gives that query's readouts no level). + tolerated_failure_probability_per_output_value: Option, +} + +impl RootLimits { + /// The limits' bits, which equality, hashing and order compare. + fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option) { + ( + self.tolerated_error_per_metric.map(f64::to_bits), + self.tolerated_failure_probability_per_output_value + .map(f64::to_bits), + ) + } +} + +impl PartialEq for RootLimits { + fn eq(&self, other: &Self) -> bool { + self.bits() == other.bits() + } +} + +impl Eq for RootLimits {} + +impl Hash for RootLimits { + fn hash(&self, state: &mut H) { + self.bits().hash(state); + } +} + +impl PartialOrd for RootLimits { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for RootLimits { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.bits().cmp(&other.bits()) + } +} + +impl AccuracyRules { + /// The rules over `inputs`; `None` when no query has an accuracy + /// target, so (K) has no rows. + pub(super) fn of(inputs: &SelectionInputs<'_>) -> Option { + let physical = inputs.physical; + let memo = &physical.logical_candidates.memo; + if memo + .queries() + .iter() + .all(|query| query.accuracy_target.is_none()) + { + return None; + } + // Pass 2 rejects a cyclic memo (`Pass2RuleError::CyclicDependencies`). + let order = memo + .topological_order() + .expect("Stage 3 reads Pass 2's acyclic memo"); + let candidate_of_execution: Vec = inputs + .selectable + .executions() + .iter() + .map(|execution| { + physical.physical_candidates[execution.candidate.0].logical_candidate_id + }) + .collect(); + let own_error_per_execution = candidate_of_execution + .iter() + .map(|&id| { + let mut own = ExecutionError::default(); + if let Some(required) = memo.candidate(id).required_accuracy() { + own.error_bound_per_metric[required.metric.index()] = required.max_error; + own.failure_probability = required.failure_probability; + } + own + }) + .collect(); + let reached_by_query_with_target = candidate_of_execution + .iter() + .map(|id| { + memo.groups()[id.group.0 as usize] + .reached_by_queries() + .iter() + .any(|&query| memo.queries()[query].accuracy_target.is_some()) + }) + .collect(); + let mut position_in_order = vec![0; memo.groups().len()]; + for (position, group) in order.iter().enumerate() { + position_in_order[group.0 as usize] = position; + } + let mut executions_producers_first: Vec = + (0..candidate_of_execution.len()).collect(); + executions_producers_first + .sort_by_key(|&c| position_in_order[candidate_of_execution[c].group.0 as usize]); + let roots = query_roots_with_target(memo) + .into_iter() + .map(|(query, root, target, allowed_metrics)| { + let failure_probability = target.failure_probability_per_output_value( + memo.queries()[query].number_of_values_per_query_evaluation, + ); + let mut tolerated_error_per_metric = [0.0; AccuracyMetric::COUNT]; + for metric in allowed_metrics { + tolerated_error_per_metric[metric.index()] = tolerated_limit(target.max_error); + } + let limits = RootLimits { + tolerated_error_per_metric, + tolerated_failure_probability_per_output_value: failure_probability + .map(tolerated_limit), + }; + ((QueryId(query), root), limits) + }) + .collect(); + Some(Self { + error_metrics: ErrorMetricsOfMemo::of(memo, &order), + candidate_of_execution, + own_error_per_execution, + reached_by_query_with_target, + executions_producers_first, + roots, + }) + } + + /// What the root check of `query`'s output reading `group` bounds it + /// by; `None` when the query has no target, so (K) checks nothing + /// there. + pub(super) fn root_limits(&self, query: QueryId, group: MemoGroupId) -> Option<&RootLimits> { + self.roots.get(&(query, group)) + } + + /// Execution `c`'s error contract in metric `m` (by + /// [`AccuracyMetric::index`]); `None` where it has none (the metric + /// rule). + fn contract(&self, c: usize, m: usize) -> Option<&ErrorContract> { + self.error_metrics + .error_contract(self.candidate_of_execution[c], AccuracyMetric::ALL[m]) + } + + /// Execution `c`'s error composed from `inputs_read`, the error read on + /// each of its inputs (input index, error), by (K)'s rule + /// (`accuracy-models.md`, "Checking each query's target in + /// selection"): its own error + /// plus, per metric, $\beta_{c,i,m}$ times each input's error in that + /// metric where the input's error reaches the output ($\beta > 0$), + /// and its own failure probability plus every input's (a union bound). + /// An input's error in a metric `c` has no contract for breaks the + /// metric rule, and one above the contract's validity cap $b_{c,i}$ + /// breaks the cap: either leaves the inputs not admitted, and adds + /// nothing in that metric for the metric rule. + /// + /// Example: `p99 / p50`'s contract passes each operand's relative + /// error with factor 1.111 and caps the denominator at 0.1: reading + /// 0.02 and 0.02 composes 0.044, admitted; reading a denominator at + /// 0.2 is not admitted. + fn compose( + &self, + c: usize, + inputs_read: impl IntoIterator, + ) -> Composition { + let mut composed = Composition { + error: self.own_error_per_execution[c], + inputs_admitted: true, + }; + for (i, read) in inputs_read { + composed.error.failure_probability += read.failure_probability; + for (m, &error) in read.error_bound_per_metric.iter().enumerate() { + if error <= 0.0 { + continue; + } + let Some(contract) = self.contract(c, m) else { + composed.inputs_admitted = false; + continue; + }; + if !contract.input_error_reaches_output(i) { + continue; + } + if contract.linear_bound_holds_up_to_per_input[i] + .is_some_and(|holds_up_to| error > tolerated_limit(holds_up_to)) + { + composed.inputs_admitted = false; + } + composed.error.error_bound_per_metric[m] += + contract.error_factor_per_input[i] * error; + } + } + composed + } + + /// Whether the selection `edges` (by position in + /// `SelectableExecutions::edges`) with the outputs `answering` (query, + /// execution answering it, in `SelectableExecutions::answering` order) + /// meets (K), exactly in f64: each execution a query with a target + /// reaches, composed ([`Self::compose`]) from the producers selected + /// on its inputs, admits its inputs' errors and, at each output of a + /// query with a target, is within the root's limits. `Err` names the + /// first query (in `answering` order) whose output misses its target. + /// + /// Example: p99 / p50 at relative 0.03 over two readouts at 0.02 does + /// not meet it (1.111 · 0.04 = 0.044), so it is `Err` with the ratio's + /// query; over two at 0.01 it does. + pub(super) fn selection_meets_targets( + &self, + inputs: &SelectionInputs<'_>, + edges: &[usize], + answering: &[(QueryId, usize)], + ) -> Result<(), QueryId> { + let selectable = inputs.selectable; + let count = selectable.executions().len(); + let mut reads: Vec> = vec![Vec::new(); count]; + for &position in edges { + let edge = selectable.edges()[position]; + let input = inputs.physical.edge(edge.edge).input_index_of_consumer; + reads[edge.consumer].push((input, edge.producer)); + } + let mut composed: Vec> = vec![None; count]; + let mut on_walk = vec![false; count]; + let memo = &inputs.physical.logical_candidates.memo; + for (&(_, group), &(query, v)) in memo.output_groups().iter().zip(answering) { + // Compose `v` and every execution below it, producers first. + let mut pending: Vec<(usize, bool)> = vec![(v, false)]; + while let Some((c, producers_done)) = pending.pop() { + if composed[c].is_some() { + continue; + } + if !producers_done { + if on_walk[c] { + // A cycle, which (E) forbids. + return Err(query); + } + on_walk[c] = true; + pending.push((c, true)); + pending.extend( + reads[c] + .iter() + .filter(|&&(_, u)| composed[u].is_none()) + .map(|&(_, u)| (u, false)), + ); + continue; + } + if !self.reached_by_query_with_target[c] { + // No query with a target reaches it: (K) has no rows + // for it, nor for its producers. + composed[c] = Some(ExecutionError::default()); + continue; + } + let Some(inputs_read) = reads[c] + .iter() + .map(|&(i, u)| Some((i, composed[u]?))) + .collect::>>() + else { + return Err(query); + }; + let composition = self.compose(c, inputs_read); + if !composition.inputs_admitted { + return Err(query); + } + composed[c] = Some(composition.error); + } + let Some(limits) = self.root_limits(query, group) else { + continue; + }; + let error = composed[v].expect("each answering execution is composed"); + if error + .error_bound_per_metric + .iter() + .zip(limits.tolerated_error_per_metric) + .any(|(&error, limit)| error > limit) + || limits + .tolerated_failure_probability_per_output_value + .is_some_and(|limit| error.failure_probability > limit) + { + return Err(query); + } + } + Ok(()) + } +} + +/// What (K) is built over, per execution (by position in +/// `SelectableExecutions::executions`), before any variable: its +/// selectable input edges and upper bounds on the error and failure +/// probability any selection gives it (the big-M of (K)). +/// +/// Only an execution that is its own original (not in a copy of an +/// equivalent context, [`SharedVariables`]) and that a query with an +/// accuracy target reaches has rows: a copy's rows would repeat its +/// original's on the same variables. An execution whose inputs carry no +/// error under any selection (a readout, whose state is exact) has a +/// constant error, its own, and no variable: its consumers read it as a +/// constant times $y$ ([`ErrorTerm`]). +/// +/// Example: B = p99 / p50 at relative 0.03, both read from DDSketch +/// readouts required at relative 0.02 or 0.01: each readout's own error is +/// its requirement, the ratio adds none and passes relative error on with +/// factor 1/(1 − 0.1) per input, so its ceiling is 1.111 (0.02 + 0.02). +pub(super) struct ErrorModel<'a> { + /// The rules (K) applies. + rules: &'a AccuracyRules, + /// Whether (K) has rows for the execution. + has_rows: Vec, + /// Per input, the selectable edges into the execution that have a + /// $y$: (position in `SelectableExecutions::edges`, producer + /// execution); empty for an execution without rows. + selectable_edges_per_input: Vec>>, + /// $M$: the largest error and failure probability any selection gives + /// the execution. + ceiling: Vec, + /// Whether the execution's error is constant: its inputs carry none + /// under any selection, so its ceilings are its own error. + error_is_constant: Vec, +} + +/// An execution's error in one metric, or its failure probability, as +/// (K)'s rows read it: a constant (a readout's +/// own requirement) or a variable with its ceiling. +#[derive(Clone, Copy)] +enum ErrorTerm { + Constant(f64), + Variable { variable: Variable, ceiling: f64 }, +} + +impl<'a> ErrorModel<'a> { + /// The model over `inputs` and `rules`, with `y` the edge variables + /// kept and `shared` the copies of equivalent contexts. + pub(super) fn new( + inputs: &SelectionInputs<'_>, + rules: &'a AccuracyRules, + y: &BTreeMap, + shared: &SharedVariables, + ) -> Self { + let selectable = inputs.selectable; + let count = selectable.executions().len(); + let has_rows: Vec = (0..count) + .map(|index| { + rules.reached_by_query_with_target[index] + && shared.original_of_execution[index] == index + }) + .collect(); + let selectable_edges_of = |index: usize| -> Vec> { + if !has_rows[index] { + return Vec::new(); + } + (0..selectable.input_count(index)) + .map(|input| { + selectable + .edges_into(index, input) + .iter() + .filter(|position| y.contains_key(position)) + .map(|&position| (position, selectable.edges()[position].producer)) + .collect() + }) + .collect() + }; + let mut model = Self { + rules, + selectable_edges_per_input: (0..count).map(selectable_edges_of).collect(), + has_rows, + ceiling: vec![ExecutionError::default(); count], + error_is_constant: vec![false; count], + }; + model.set_ceilings(); + model.error_is_constant = (0..count) + .map(|c| { + model.has_rows[c] + && model.selectable_edges_per_input[c] + .iter() + .flatten() + .all(|&(_, u)| model.ceiling[u].is_zero()) + }) + .collect(); + model + } + + /// Execution `u`'s error in metric `m` (by [`AccuracyMetric::index`]), + /// `None` when it is 0 under every selection. + fn error_bound(&self, accuracy: &AccuracyVariables, u: usize, m: usize) -> Option { + let ceiling = self.ceiling[u].error_bound_per_metric[m]; + match (ceiling > 0.0, self.error_is_constant[u]) { + (false, _) => None, + (true, true) => Some(ErrorTerm::Constant(ceiling)), + (true, false) => Some(ErrorTerm::Variable { + variable: accuracy.error_bound[&(u, m)], + ceiling, + }), + } + } + + /// Execution `u`'s failure probability; `None` when it is 0 + /// under every selection. + fn failure_probability(&self, accuracy: &AccuracyVariables, u: usize) -> Option { + let ceiling = self.ceiling[u].failure_probability; + match (ceiling > 0.0, self.error_is_constant[u]) { + (false, _) => None, + (true, true) => Some(ErrorTerm::Constant(ceiling)), + (true, false) => Some(ErrorTerm::Variable { + variable: accuracy.failure_probability[&u], + ceiling, + }), + } + } + + /// The ceilings, in one pass over the executions with rows, producers + /// first ([`AccuracyRules::executions_producers_first`]): each + /// execution composed ([`AccuracyRules::compose`]) from the largest + /// ceiling of the producers selectable on each input. Every selectable + /// edge reads a group the memo orders before the consumer's (the memo + /// is acyclic, so no group reads itself), so each producer's ceiling is + /// final when its consumer reads it. + fn set_ceilings(&mut self) { + let rules = self.rules; + for &c in &rules.executions_producers_first { + if !self.has_rows[c] { + continue; + } + let largest_per_input: Vec<(usize, ExecutionError)> = self.selectable_edges_per_input + [c] + .iter() + .enumerate() + .map(|(i, edges)| { + let largest = + edges + .iter() + .fold(ExecutionError::default(), |largest, &(_, u)| { + debug_assert_ne!( + rules.candidate_of_execution[u].group, + rules.candidate_of_execution[c].group, + "a selectable edge reads another group" + ); + largest.largest(self.ceiling[u]) + }); + (i, largest) + }) + .collect(); + // A ceiling bounds every selection, admitted or not. + self.ceiling[c] = rules.compose(c, largest_per_input).error; + } + } +} + +/// The accuracy variables of constraint (K), all continuous, in +/// each only for an execution with rows +/// ([`ErrorModel`]) whose error is not constant and whose ceiling is above +/// 0, by execution position. +pub(super) struct AccuracyVariables { + /// $E_{\kappa,c,m}$: the execution's error bound in metric $m$ (by + /// [`AccuracyMetric::index`]). + error_bound: HashMap<(usize, usize), Variable>, + /// $D_{\kappa,c}$: its failure probability. + failure_probability: HashMap, +} + +impl AccuracyVariables { + /// $E$ and $D$ over `model`, added to `problem`. + pub(super) fn new(model: &ErrorModel<'_>, problem: &mut ProblemVariables) -> Self { + let mut variables = Self { + error_bound: HashMap::new(), + failure_probability: HashMap::new(), + }; + for c in + (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) + { + for (m, &ceiling) in model.ceiling[c].error_bound_per_metric.iter().enumerate() { + if ceiling > 0.0 { + variables + .error_bound + .insert((c, m), problem.add(variable().min(0))); + } + } + if model.ceiling[c].failure_probability > 0.0 { + variables + .failure_probability + .insert(c, problem.add(variable().min(0))); + } + } + variables + } + + /// The number of $E$ and $D$ variables. + #[cfg(test)] + pub(super) fn variable_count(&self) -> usize { + self.error_bound.len() + self.failure_probability.len() + } +} + +/// (K) (`accuracy-models.md`, "Checking each query's target in +/// selection"), per execution $c$ with rows ([`ErrorModel`]): +/// - $E_{c,m} \ge \varepsilon_c[m = m_c] + \sum_i \beta_{c,i,m} +/// F_{c,i,m}$ and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per +/// selectable edge, $F_{c,i,m} \le b_{c,i}$ where the contract's linear +/// bound holds only up to $b_{c,i}$ (a ratio's denominator), and +/// $E_{u,m} \le M_{u,m}(1 - y_{u,c,i})$ when $c$ has no contract in $m$ +/// (the metric rule on the selected path); an input whose error does not +/// reach the output ($\beta = 0$, a count's) passes none; +/// - $D_c \ge \delta_c + \sum_i H_{c,i}$, $H_{c,i} \ge D_u - M_u(1 - +/// y_{u,c,i})$ (a union bound); +/// - per query $q$ with a target, output group $g$ its root reads and +/// execution $v$ answering it, $E_{v,m} \le \varepsilon_q[m \in +/// \text{allowed}] + M(1 - z_{q,v})$, the metrics $g$ allows at the root +/// ([`query_roots_with_target`]: the target's at an operator root, none +/// for the operands of a scalar expression no contract covers), and +/// $D_v \le \delta_q + M(1 - z_{q,v})$ with $\delta_q$ per output value +/// (`AccuracyTarget::failure_probability_per_output_value`); no $D$ row +/// without one ([`RootLimits`], each limit widened by +/// [`tolerated_limit`]). +/// +/// It is [`AccuracyRules::compose`]'s rule over variables: the same own +/// error, factors, metric rule and validity caps. +/// +/// $F$ and $H$ are added to `problem` here. A copy of an equivalent context +/// and its queries have no rows: they share their original's variables, +/// and the copy's rows would be its original's +/// ([`super::equivalent_contexts::copies_in_equivalent_contexts`] pairs +/// queries only with equal [`RootLimits`] per output). +pub(super) fn accuracy_constraints( + inputs: &SelectionInputs<'_>, + model: &ErrorModel<'_>, accuracy: &AccuracyVariables, + y: &BTreeMap, + z: &BTreeMap<(QueryId, usize), Variable>, + shared: &SharedVariables, + problem: &mut ProblemVariables, ) -> Vec { - let _ = (physical, selection, accuracy); - todo!("#1207 PR 8: constraint (K)") + let rules = model.rules; + let mut constraints = Vec::new(); + for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) + { + // Σ_i β_{c,i,m} F_{c,i,m} per metric, and Σ_i H_{c,i}. + let mut error_of_inputs: [Expression; AccuracyMetric::COUNT] = Default::default(); + let mut failure_probability_of_inputs = Expression::default(); + for (i, edges) in model.selectable_edges_per_input[c].iter().enumerate() { + for (m, error_of_input) in error_of_inputs.iter_mut().enumerate() { + let read: Vec<(Variable, ErrorTerm)> = edges + .iter() + .filter_map(|&(position, u)| { + Some((y[&position], model.error_bound(accuracy, u, m)?)) + }) + .collect(); + if read.is_empty() { + continue; + } + let Some(contract) = rules.contract(c, m) else { + // The metric rule: no error in m reaches c. + for (y, error) in read { + constraints.push(match error { + ErrorTerm::Constant(_) => constraint!(y <= 0), + ErrorTerm::Variable { variable, ceiling } => { + constraint!(variable <= ceiling * (1 - y)) + } + }); + } + continue; + }; + if !contract.input_error_reaches_output(i) { + continue; + } + let of_input = chosen_producers(&read, &mut constraints, problem); + if let Some(holds_up_to) = contract.linear_bound_holds_up_to_per_input[i] { + let holds_up_to = tolerated_limit(holds_up_to); + constraints.push(constraint!(of_input.clone() <= holds_up_to)); + } + *error_of_input += contract.error_factor_per_input[i] * of_input; + } + // H_{c,i} counts the failure of the one producer selected on + // input i, not of each readout value an output value depends + // on: `max(q)` over 1,000 series reads 1,000 readout values, + // and D here adds the readout's δ once. That is sound only + // because Pass 2's levels already split the query's δ over + // the readout values its output depends on + // (`readout_caps::query_levels`), so each readout's δ is the + // query's over 1,000; the root row checks the composed sum + // against the per-output-value δ. + let read: Vec<(Variable, ErrorTerm)> = edges + .iter() + .filter_map(|&(position, u)| { + Some((y[&position], model.failure_probability(accuracy, u)?)) + }) + .collect(); + if !read.is_empty() { + failure_probability_of_inputs += chosen_producers(&read, &mut constraints, problem); + } + } + let own = rules.own_error_per_execution[c]; + for (m, of_inputs) in error_of_inputs.into_iter().enumerate() { + if let Some(&error_bound) = accuracy.error_bound.get(&(c, m)) { + let own = own.error_bound_per_metric[m]; + constraints.push(constraint!(error_bound >= own + of_inputs)); + } + } + if let Some(&failure_probability) = accuracy.failure_probability.get(&c) { + let own = own.failure_probability; + constraints.push(constraint!( + failure_probability >= own + failure_probability_of_inputs + )); + } + } + let memo = &inputs.physical.logical_candidates.memo; + for (&(query, group), (query_id, executions)) in memo + .output_groups() + .iter() + .zip(inputs.selectable.answering()) + { + debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); + if shared.original_of_query[query] != *query_id { + continue; + } + let Some(limits) = rules.root_limits(*query_id, group) else { + continue; + }; + for &v in executions { + let z = z[&(*query_id, v)]; + for (m, &limit) in limits.tolerated_error_per_metric.iter().enumerate() { + constraints.extend( + model + .error_bound(accuracy, v, m) + .and_then(|error| at_most_when_answering(error, limit, z)), + ); + } + if let (Some(limit), Some(failure_probability)) = ( + limits.tolerated_failure_probability_per_output_value, + model.failure_probability(accuracy, v), + ) { + constraints.extend(at_most_when_answering(failure_probability, limit, z)); + } + } + } + constraints +} + +/// The error (or failure probability) one input of a consumer reads from +/// the producer selected on it, `read` listing each producer's with its +/// $y$: $\sum_u e_u y_u$ over the producers with a constant error (at most +/// one $y$ of an input is 1, constraint (A)), and over the others a +/// variable $F \ge E_u - M_u(1 - y_u)$, also at least that sum. +fn chosen_producers( + read: &[(Variable, ErrorTerm)], + constraints: &mut Vec, + problem: &mut ProblemVariables, +) -> Expression { + let constant: Expression = read + .iter() + .filter_map(|&(y, term)| match term { + ErrorTerm::Constant(value) => Some(value * y), + ErrorTerm::Variable { .. } => None, + }) + .sum(); + if read + .iter() + .all(|(_, term)| matches!(term, ErrorTerm::Constant(_))) + { + return constant; + } + let of_input = problem.add(variable().min(0)); + constraints.push(constraint!(of_input >= constant)); + for &(y, term) in read { + if let ErrorTerm::Variable { variable, ceiling } = term { + constraints.push(constraint!(of_input >= variable - ceiling * (1 - y))); + } + } + of_input.into() +} + +/// The root row bounding `term` by `limit` when $z = 1$: none when the +/// ceiling is within it; $z = 0$ for a constant above it. +fn at_most_when_answering(term: ErrorTerm, limit: f64, z: Variable) -> Option { + match term { + ErrorTerm::Constant(value) => (value > limit).then(|| constraint!(z <= 0)), + ErrorTerm::Variable { variable, ceiling } => { + (limit < ceiling).then(|| constraint!(variable <= limit + (ceiling - limit) * (1 - z))) + } + } } diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs index f48352bb0..0a5113489 100644 --- a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use asap_types::accuracy::AccuracyTarget; use asap_types::stages::{ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId}; +use super::accuracy::{AccuracyRules, RootLimits}; use super::model::SelectionInputs; use super::COST_RELATIVE_TOLERANCE; use crate::selectable::QueryId; @@ -73,6 +74,13 @@ struct QuerySignature<'a> { /// Its accuracy target, by index among the workload's distinct targets /// ([`AccuracyTarget`] holds floats: metric, ε, δ and scope all equal). accuracy_target_index: usize, + /// Per output, what (K)'s root check bounds it by + /// ([`AccuracyRules::root_limits`]); `None` without a target. Equal + /// limits give equal root rows: two queries at δ = 0.01 per value + /// returning 10 and 100 values have equal limits, over a whole result + /// they do not; an operator root and an operand of `scalar(...) * 2` + /// allow different metrics. + root_limits_per_output: Vec>, } /// Which executions, edges and queries share variables @@ -82,8 +90,9 @@ struct QuerySignature<'a> { /// they have the same [`ContextKey`] (the same candidates, kept alike, the /// same memo edges from producers that correspond, in the context or the /// same execution outside every evaluation context, and queries that pair -/// up with the same output executions, exact-only groups, span and -/// accuracy target, so the accuracy constraint holds alike in both) and +/// up with the same output executions, exact-only groups, span, accuracy +/// target and root limits of (K) at each output (`rules`), so the accuracy +/// constraint (K) has the same rows in both) and /// every cost the MILP reads in one (of a kept execution, or of an edge /// between two kept ones) is the same multiple $\alpha > 0$ of the /// other's, to [`COST_RELATIVE_TOLERANCE`] of the copy's largest such @@ -107,9 +116,10 @@ struct QuerySignature<'a> { /// Example: the same queries every 60 s and every 300 s (one user each): /// the 300 s context's executions cost 1/5 of the 60 s ones' and share /// their variables. -pub(crate) fn copies_in_equivalent_contexts( - inputs: &SelectionInputs<'_>, +pub(crate) fn copies_in_equivalent_contexts<'a>( + inputs: &SelectionInputs<'a>, kept: &[bool], + rules: Option<&'a AccuracyRules>, ) -> SharedVariables { let selectable = inputs.selectable; let costs = inputs.costs; @@ -131,9 +141,18 @@ pub(crate) fn copies_in_equivalent_contexts( own.push(index); } } - let mut outputs_of: BTreeMap> = BTreeMap::new(); - for (query, answering) in selectable.answering() { - outputs_of.entry(*query).or_default().push(answering); + let memo = &inputs.physical.logical_candidates.memo; + // Per query, each output's answering executions and its root limits. + let mut outputs_of: BTreeMap> = BTreeMap::new(); + for (&(query, group), (query_id, answering)) in + memo.output_groups().iter().zip(selectable.answering()) + { + debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); + let limits = rules.and_then(|rules| rules.root_limits(*query_id, group)); + outputs_of + .entry(*query_id) + .or_default() + .push((answering, limits)); } // Each evaluation context's queries; a query without an output has // none. @@ -172,11 +191,11 @@ pub(crate) fn copies_in_equivalent_contexts( }) }) .collect(); - let signature = |context: ExecutionContextId, query: QueryId| -> Option> { + let signature = |context: ExecutionContextId, query: QueryId| -> Option> { // A query has an evaluation context exactly when it has an output reference. let outputs = outputs_of[&query] .iter() - .map(|answering| { + .map(|(answering, _)| { answering .iter() .map(|&index| in_context(context, index)) @@ -185,6 +204,10 @@ pub(crate) fn copies_in_equivalent_contexts( .collect::>>()?; Some(QuerySignature { answering_executions_per_output: outputs, + root_limits_per_output: outputs_of[&query] + .iter() + .map(|&(_, limits)| limits) + .collect(), exact_only_groups: inputs .physical .logical_candidates @@ -196,7 +219,7 @@ pub(crate) fn copies_in_equivalent_contexts( }; // `context`'s key, and its queries in the order of their signatures in // it. - let key_of = |context: ExecutionContextId| -> Option<(ContextKey<'_>, Vec)> { + let key_of = |context: ExecutionContextId| -> Option<(ContextKey<'a>, Vec)> { let executions = executions_of[&context] .iter() .map(|&index| { @@ -297,7 +320,7 @@ pub(crate) fn copies_in_equivalent_contexts( .all(|&(a, b)| (b - alpha * a).abs() <= tolerance) }; // Per key, the originals, each with its queries in signature order. - let mut originals_of_key: HashMap, Vec<(ExecutionContextId, Vec)>> = + let mut originals_of_key: HashMap, Vec<(ExecutionContextId, Vec)>> = HashMap::new(); for © in executions_of.keys() { let Some((key, copy_queries)) = key_of(copy) else { diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 1394a0982..26b7fe166 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -7,11 +7,9 @@ //! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_rows`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | //! | retention (section 5) | $h_v$ | (R) [`super::retention::retention_constraints`] | //! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | -//! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | +//! | accuracy (`accuracy-models.md`, constraint (K)), per execution | $E_{\kappa,c,m}$, $D_{\kappa,c}$ ([`AccuracyVariables`]); $F_{\kappa,c,i,m}$, $H_{\kappa,c,i}$ | (K) [`accuracy_constraints`], over [`ErrorModel`] | //! -//! [`solve_selection`] solves it without (K), whose variables and -//! constraint are in [`super::accuracy`], `todo!()` until it is added. The -//! strongly connected components (E) needs are in [`super::graph`]; the +//! The strongly connected components (E) needs are in [`super::graph`]; the //! executions left out before scaling, those dearer than a feasible plan //! and those their `OneEvaluation` twin dominates, in //! [`super::upper_bound`] and [`super::dominance`]. @@ -28,6 +26,7 @@ use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; +use super::accuracy::{accuracy_constraints, AccuracyRules, AccuracyVariables, ErrorModel}; use super::dominance::dominated_by_one_evaluation_twin; use super::equivalent_contexts::{copies_in_equivalent_contexts, SharedVariables}; use super::exact_only::{exact_only_constraints, exact_only_variables}; @@ -60,9 +59,9 @@ pub(crate) struct SelectionVariables { x: Vec, /// $y_{\kappa,u,c,i}$, by edge, on the edges costed between executions /// kept, in edge order. - y: BTreeMap, + pub(super) y: BTreeMap, /// $z_{q,v}$, by query and the execution answering it, in key order. - z: BTreeMap<(QueryId, usize), Variable>, + pub(super) z: BTreeMap<(QueryId, usize), Variable>, /// $t$, by execution: only for executions on a cycle of selectable /// edges (a strongly connected component); `None` elsewhere. t: Vec>, @@ -82,11 +81,12 @@ pub(crate) struct SelectionVariables { /// fixed at 0 for an execution that is not `kept` (not costed, or dearer /// than a feasible plan), and no $y$ on an edge of one. The executions, /// edges and queries of a copy of an equivalent evaluation context take the -/// original's $x$, $y$ and $z$ ([`copies_in_equivalent_contexts`]), so -/// their costs add up on one variable. +/// original's $x$, $y$ and $z$ (`shared`, [`copies_in_equivalent_contexts`]), +/// so their costs add up on one variable. fn selection_variables( inputs: &SelectionInputs<'_>, kept: &[bool], + shared: &SharedVariables, components: &Components, problem: &mut ProblemVariables, ) -> SelectionVariables { @@ -95,7 +95,7 @@ fn selection_variables( original_of_execution, original_of_edge, original_of_query, - } = copies_in_equivalent_contexts(inputs, kept); + } = shared; let mut x: Vec = Vec::with_capacity(kept.len()); for (index, &kept) in kept.iter().enumerate() { let variable = match (original_of_execution[index], kept) { @@ -364,7 +364,13 @@ pub(crate) struct SolvedSelection { pub(crate) variable_count: usize, pub(crate) constraint_count: usize, pub(crate) construction_ms: u64, + /// The time of every HiGHS solve, summed. pub(crate) solve_ms: u64, + /// How many times HiGHS solved the MILP: 1, or 2 when the selection at + /// HiGHS's default tolerances missed an accuracy target when checked + /// exactly and the MILP was solved again at + /// [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`]. + pub(crate) solve_count: usize, } /// A selected $y_{\kappa,u,c,i}$: edge `edge` from execution `producer` @@ -383,14 +389,31 @@ pub(crate) enum SelectionMilpError { Infeasible, /// The solver failed. Solver(String), + /// The solver's selection misses `query`'s accuracy target when + /// checked exactly: its limits, such as a failure probability of + /// 1e-12 per output value (δ = 0.01 over 1e10 values), are below the + /// solver's least feasibility tolerance, + /// [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`], so (K) cannot be trusted. + TargetBelowSolverPrecision { + query: QueryId, + /// The target's δ per output value; `None` without a δ. + failure_probability_per_output_value: Option, + }, } +/// The smallest primal and MIP feasibility tolerance HiGHS accepts, 1e-10 +/// (absolute; HiGHS's default is 1e-7). A second solve uses it when the +/// selection at the default tolerances misses an accuracy target: a +/// per-value δ of 1e-6 (δ = 0.01 over 1e4 values) may be passed by up to +/// 1e-7 at the default, but only by 1e-10 here. +const LEAST_HIGHS_FEASIBILITY_TOLERANCE: f64 = 1e-10; + /// Builds and solves the MILP: minimize /// $\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} /// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} /// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a /// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], -/// [`raw_data_disk_rows`], (R) and (X), with HiGHS at gap 0 on one +/// [`raw_data_disk_rows`], (R), (X) and (K), with HiGHS at gap 0 on one /// thread. The executions dearer than a feasible plan /// ([`dearer_than_a_feasible_plan`]) or dominated by their `OneEvaluation` /// twin ([`dominated_by_one_evaluation_twin`]) are left out first (and a @@ -398,21 +421,45 @@ pub(crate) enum SelectionMilpError { /// [`copies_in_equivalent_contexts`]), with /// the $h_v$ of a candidate whose /// executions are all left out, then the coefficients kept are scaled so -/// the largest is 1,000 (HiGHS's tolerances are absolute). +/// the largest is 1,000 (HiGHS's tolerances are absolute). HiGHS solves +/// at its default tolerances and the selection is then checked exactly +/// ([`AccuracyRules::selection_meets_targets`]); a selection that misses a +/// target is solved once more at [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`], +/// and if that one misses too, the error is +/// [`SelectionMilpError::TargetBelowSolverPrecision`]. pub(crate) fn solve_selection( inputs: &SelectionInputs<'_>, ) -> Result { let started = Instant::now(); let costs = inputs.costs; - let kept: Vec = dearer_than_a_feasible_plan(inputs) + let rules = AccuracyRules::of(inputs); + let kept: Vec = dearer_than_a_feasible_plan(inputs, rules.as_ref()) .iter() .zip(dominated_by_one_evaluation_twin(inputs)) .zip(&costs.per_execution) .map(|((&dearer, dominated), cost)| cost.is_ok() && !dearer && !dominated) .collect(); let components = Components::of(inputs); + let shared = copies_in_equivalent_contexts(inputs, &kept, rules.as_ref()); let mut problem = ProblemVariables::new(); - let variables = selection_variables(inputs, &kept, &components, &mut problem); + let variables = selection_variables(inputs, &kept, &shared, &components, &mut problem); + // (K), only when some query has an accuracy target. + let accuracy_rows = rules + .as_ref() + .map(|rules| { + let error_model = ErrorModel::new(inputs, rules, &variables.y, &shared); + let accuracy = AccuracyVariables::new(&error_model, &mut problem); + accuracy_constraints( + inputs, + &error_model, + &accuracy, + &variables.y, + &variables.z, + &shared, + &mut problem, + ) + }) + .unwrap_or_default(); // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs @@ -467,6 +514,7 @@ pub(crate) fn solve_selection( &variables.z, ), exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), + accuracy_rows, ] .into_iter() .flatten() @@ -501,50 +549,90 @@ pub(crate) fn solve_selection( model.set_option("output_flag", false); model.set_option("threads", 1); let construction_ms = started.elapsed().as_millis() as u64; - let started = Instant::now(); - let solved = model - .try_solve() - .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; - let solve_ms = started.elapsed().as_millis() as u64; - use highs::{HighsModelStatus, HighsSolutionStatus}; - let columns: Vec = match solved.status() { - // No constraint: no query reads an operator (literal-only scalar - // roots), so nothing is selected. - HighsModelStatus::ModelEmpty => vec![0.0; variable_count], - HighsModelStatus::Optimal - if solved.primal_solution_status() == HighsSolutionStatus::Feasible => - { - solved.get_solution().columns().to_vec() - } - HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { - return Err(SelectionMilpError::Infeasible) - } - other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), - }; - let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; let selectable = inputs.selectable; let executions = selectable.executions(); - let selected: Vec = (0..variables.x.len()) - .filter(|&index| chosen(variables.x[index])) - .collect(); - let selected_edges: Vec = variables - .y - .iter() - .filter(|&(_, &y)| chosen(y)) - .map(|(&position, _)| position) - .collect(); - let answering: Vec<(QueryId, usize)> = selectable - .answering() - .iter() - .map(|(query, answering)| { - let execution = answering - .iter() - .copied() - .find(|&execution| chosen(variables.z[&(*query, execution)])) - .expect("(C): one output per query output reference"); - (*query, execution) - }) - .collect(); + let mut solve_ms = 0; + let mut solve_count = 0; + let (selected, selected_edges, answering) = loop { + let started = Instant::now(); + let solved = model + .try_solve() + .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; + solve_ms += started.elapsed().as_millis() as u64; + solve_count += 1; + use highs::{HighsModelStatus, HighsSolutionStatus}; + let columns: Vec = match solved.status() { + // No constraint: no query reads an operator (literal-only scalar + // roots), so nothing is selected. + HighsModelStatus::ModelEmpty => vec![0.0; variable_count], + HighsModelStatus::Optimal + if solved.primal_solution_status() == HighsSolutionStatus::Feasible => + { + solved.get_solution().columns().to_vec() + } + HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { + return Err(SelectionMilpError::Infeasible) + } + other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), + }; + let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; + let selected: Vec = (0..variables.x.len()) + .filter(|&index| chosen(variables.x[index])) + .collect(); + let selected_edges: Vec = variables + .y + .iter() + .filter(|&(_, &y)| chosen(y)) + .map(|(&position, _)| position) + .collect(); + let answering: Vec<(QueryId, usize)> = selectable + .answering() + .iter() + .map(|(query, answering)| { + let execution = answering + .iter() + .copied() + .find(|&execution| chosen(variables.z[&(*query, execution)])) + .expect("(C): one output per query output reference"); + (*query, execution) + }) + .collect(); + // (K) holds only to the solver's tolerance: at the default 1e-7, a + // per-value δ of 1e-6 may be missed by a tenth, and at 1e-10 a δ + // near or below 1e-10 by orders of magnitude. The exact check sends + // a selection that misses a target to a second solve at + // LEAST_HIGHS_FEASIBILITY_TOLERANCE, and refuses one that misses + // after it rather than return a plan that may not meet its target. + let Some(Err(query)) = rules + .as_ref() + .map(|rules| rules.selection_meets_targets(inputs, &selected_edges, &answering)) + else { + break (selected, selected_edges, answering); + }; + if solve_count == 2 { + let memo = &inputs.physical.logical_candidates.memo; + let memo_query = &memo.queries()[query.0]; + return Err(SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value: memo_query.accuracy_target.as_ref().and_then( + |target| { + target.failure_probability_per_output_value( + memo_query.number_of_values_per_query_evaluation, + ) + }, + ), + }); + } + model = solved.into(); + model.set_option( + "primal_feasibility_tolerance", + LEAST_HIGHS_FEASIBILITY_TOLERANCE, + ); + model.set_option( + "mip_feasibility_tolerance", + LEAST_HIGHS_FEASIBILITY_TOLERANCE, + ); + }; let (cost_per_second_without_request_overhead, retention_ms) = selection_cost_per_second(inputs, &selected, &selected_edges, &answering); Ok(SolvedSelection { @@ -571,6 +659,7 @@ pub(crate) fn solve_selection( constraint_count, construction_ms, solve_ms, + solve_count, }) } diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index f5b68ae7b..ee8b8170e 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -4,23 +4,31 @@ use std::collections::{BTreeSet, HashMap}; -use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::scalar::ExprSemantics; +use asap_types::ir::scalar::ScalarValue; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; +use asap_types::ir::{QueryRoot, ScalarExpr}; use asap_types::stages::{ - CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, + CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalRequirements, + Memo, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, RequiredAccuracy, }; use asap_types::stages::{Lifetime, PhysicalProperties, StorageMedium}; -use asap_types::summary_type::{ExactKind, ExactParams, SketchAlgorithm, SummaryUpdate}; +use asap_types::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, SummaryUpdate, +}; use asap_types::test_support::memo_builders::{ - group, own_candidate, quantile_over_time_readouts, query, scan_group, try_candidate, + group, own_candidate, quantile_over_time_readouts, query, scan_group, sketch_state, + try_candidate, }; use asap_types::test_support::memo_fixtures; +use asap_types::test_support::memo_fixtures::arithmetic; use asap_types::workload::DataWorkload; use asap_types::workload::{ Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, @@ -29,6 +37,7 @@ use asap_types::workload::{ use super::*; use crate::cost::ExecutionCosts; +use crate::milp::accuracy::{AccuracyRules, AccuracyVariables, ErrorModel}; use crate::plan::plan_of; use crate::retention::{edge_retention_need_ms, least_retention_ms_per_execution}; use crate::selectable::{QueryId, SelectableEdge}; @@ -767,7 +776,8 @@ fn contexts_whose_queries_read_different_groups_are_not_copies() { selectable: &selectable, costs: &costs, }; - let shared = copies_in_equivalent_contexts(&inputs, &vec![true; selectable.executions().len()]); + let shared = + copies_in_equivalent_contexts(&inputs, &vec![true; selectable.executions().len()], None); assert!(shared .original_of_execution .iter() @@ -859,7 +869,7 @@ fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { selectable: &selectable, costs: &costs, }; - let left_out = dearer_than_a_feasible_plan(&inputs); + let left_out = dearer_than_a_feasible_plan(&inputs, None); let unused: Vec = selectable .executions() .iter() @@ -893,12 +903,15 @@ fn the_bound_counts_the_least_retention_an_execution_pays() { let entries = every_10s(1); let (selectable, costs) = selection_inputs(&physical, &entries, &[1.0, 2.0, 1.0], &[], &[(1, 1e6)]); - let left_out = dearer_than_a_feasible_plan(&SelectionInputs { - physical: &physical, - entries: &entries, - selectable: &selectable, - costs: &costs, - }); + let left_out = dearer_than_a_feasible_plan( + &SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }, + None, + ); let left_out: Vec = selectable .executions() .iter() @@ -1009,7 +1022,7 @@ fn a_context_equivalent_to_another_shares_its_variables() { selectable: &selectable, costs, }; - copies_in_equivalent_contexts(&inputs, &kept) + copies_in_equivalent_contexts(&inputs, &kept, None) }; let SharedVariables { original_of_execution, @@ -1059,6 +1072,7 @@ fn a_context_equivalent_to_another_shares_its_variables() { costs: &costs, }, &kept, + None, ); assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(1)]); assert!(shared @@ -1130,6 +1144,7 @@ fn contexts_whose_small_edge_costs_differ_by_rounding_are_copies() { costs: &costs, }, &vec![true; executions.len()], + None, ); assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(0)]); for (index, &original) in shared.original_of_execution.iter().enumerate() { @@ -1198,3 +1213,475 @@ fn a_left_out_execution_does_not_set_the_tolerance_of_context_copies() { ); assert_eq!(candidates(&solved), vec![0, 1, 2, 2]); } + +// Constraint (K). + +/// Quantile `q` over a DDSketch state of `m[1m]` (one build per listed α): +/// the readout group, one readout per relative-error requirement of +/// `levels` (ε, δ). +fn ddsketch_readouts( + memo: &mut Memo, + q: f64, + levels: &[(f64, f64)], +) -> (MemoGroupId, Vec) { + let m = scan(memo); + let window = range(memo, 60, m); + let state = sketch_state( + memo, + window, + SketchAlgorithm::DDSketch, + Reduction::PerEntity, + ); + let operator = memo_fixtures::readout(SketchStatistic::Quantile { q }, state); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let readouts = levels + .iter() + .map(|&(max_error, failure_probability)| { + let mut readout = CandidateForMemoGroup::new( + operator.clone(), + None, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ); + readout.input_requirements = vec![LogicalRequirements { + required_accuracy: Some(RequiredAccuracy { + metric: AccuracyMetric::Relative, + max_error, + failure_probability, + }), + }]; + memo.add_candidate(group, readout).unwrap() + }) + .collect(); + (group, readouts) +} + +/// Every candidate of `memo` at query time, `OneEvaluation`. +fn every_candidate(memo: &Memo) -> Vec<(CandidateForMemoGroupId, Lifetime)> { + memo.groups() + .iter() + .enumerate() + .flat_map(|(group, memo_group)| { + (0..memo_group.candidates.len()).map(move |index_in_group| { + ( + CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }, + ONE_EVALUATION, + ) + }) + }) + .collect() +} + +/// The physical candidate of logical candidate `id` in `physical`. +fn physical_of(physical: &PhysicalASAPMemoCandidates, id: CandidateForMemoGroupId) -> usize { + physical + .physical_candidates + .iter() + .position(|candidate| candidate.logical_candidate_id == id) + .unwrap() +} + +fn relative(max_error: f64) -> AccuracyTarget { + AccuracyTarget::max_error(max_error).accuracy_metric(AccuracyMetric::Relative) +} + +/// Query B, p99 / p50 at relative 0.03 (`accuracy-models.md`, "Example: +/// allocation in a ratio"), over readouts required at relative +/// 0.02 (cost 1) or 0.01 (cost 3) each: 1.111 (0.02 + 0.02) = 0.044 and +/// 1.111 (0.02 + 0.01) = 0.033 are above 0.03, so (K) selects both 0.01 +/// readouts (0.022). The greedy plan reads the two 0.02 readouts (cost 2) +/// and misses the target, so it bounds nothing: as a bound it would leave +/// out each 0.01 readout (cost 3 > 2). The same ratio every 10 s and every +/// 20 s is one context's copy, whose (K) has no rows of its own and +/// selects alike. +#[test] +fn k_allocates_a_ratio_s_error_among_its_readouts() { + for entries in [vec![every(10_000)], vec![every(10_000), every(20_000)]] { + let mut memo = empty_memo(); + let levels = [(0.02, 1.0), (0.01, 1.0)]; + let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); + let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + for _ in &entries { + query(&mut memo, ratio, Some(relative(0.03))); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let mut w = vec![0.0; physical.physical_candidates.len()]; + for readouts in [&p99_readouts, &p50_readouts] { + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + } + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + // Each cost in the 20 s context is half the 10 s one's. + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if let asap_types::stages::ExecutionContext::Evaluation { grid } = + selectable.context(selectable.executions()[index].context) + { + if grid.period_ms == 20_000 { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + } + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let solved = solve_selection(&inputs).unwrap(); + let selected = candidates(&solved); + for readouts in [&p99_readouts, &p50_readouts] { + let chosen = physical_of(&physical, readouts[1]); + let not_chosen = physical_of(&physical, readouts[0]); + assert_eq!( + selected.iter().filter(|&&v| v == chosen).count(), + entries.len(), + "{selected:?}" + ); + assert!(!selected.contains(¬_chosen), "{selected:?}"); + } + // (K)'s E and D exist only in the 10 s context (the 20 s one is + // its copy), and only for the ratio: a readout's error is fixed. + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs).unwrap(); + let shared = copies_in_equivalent_contexts(&inputs, &kept, Some(&rules)); + let mut problem = ProblemVariables::new(); + let variables = selection_variables( + &inputs, + &kept, + &shared, + &Components::of(&inputs), + &mut problem, + ); + let model = ErrorModel::new(&inputs, &rules, &variables.y, &shared); + let accuracy = AccuracyVariables::new(&model, &mut problem); + let one_context = 2; + assert_eq!(accuracy.variable_count(), one_context, "{entries:?}"); + } +} + +/// A query reading one readout group of quantile 0.5 whose readouts have +/// `levels`, rooted at `root` over it, with `target` and $n_q$ +/// `values_per_evaluation`: whether some plan meets (K). +fn plans( + target: Option, + levels: &[(f64, f64)], + values_per_evaluation: Option, + root: fn(MemoGroupId) -> QueryRoot, +) -> bool { + let mut memo = empty_memo(); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, levels); + memo.add_query(MemoQuery { + root: root(readout_group), + accuracy_target: target, + number_of_values_per_query_evaluation: values_per_evaluation, + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + match solve(&physical, &every_10s(1), &w, &[], &[]).0 { + Ok(_) => true, + Err(SelectionMilpError::Infeasible) => false, + Err(other) => panic!("{other:?}"), + } +} + +/// The root accepts error only in the target's metric: a readout off by +/// relative 0.01 meets relative 0.01, not relative 0.005 nor rank 0.01; a +/// target naming no metric accepts any. +#[test] +fn k_accepts_error_only_in_the_target_s_metric() { + let level = [(0.01, 1.0)]; + let operator = QueryRoot::Operator; + assert!(plans(Some(relative(0.01)), &level, Some(1), operator)); + assert!(!plans(Some(relative(0.005)), &level, Some(1), operator)); + let rank = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank); + assert!(!plans(Some(rank), &level, Some(1), operator)); + let any = AccuracyTarget::max_error(0.01); + assert!(plans(Some(any), &level, Some(1), operator)); +} + +/// A whole-result failure probability 0.01 over 10 values is 1e-3 per +/// value, which a readout failing with probability 1e-3 meets; over 100 +/// values (1e-4 per value) it does not. With the count unknown no +/// per-value δ follows, and (K) has no row for it (Pass 2 gives such a +/// query's readouts no level). +#[test] +fn k_divides_a_whole_result_failure_probability_over_the_values() { + let level = [(0.01, 1e-3)]; + let target = relative(0.01).failure_probability(0.01); + let operator = QueryRoot::Operator; + assert!(plans(Some(target.clone()), &level, Some(10), operator)); + assert!(!plans(Some(target.clone()), &level, Some(100), operator)); + assert!(plans(Some(target), &level, None, operator)); +} + +/// `scalar(v)` returns the value it reads, so its root accepts the +/// target's metric; `scalar(v) * 2`'s operand is read exactly, as no error +/// contract covers the scalar expression; without a target either reads +/// the readout. +#[test] +fn a_scalar_root_accepts_the_target_s_metric_only_where_it_returns_its_operand() { + let level = [(0.01, 1.0)]; + let scalar = |group| QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(group)); + let doubled = |group| { + QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::PromqlScalarFromVector(group)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }) + }; + assert!(plans(Some(relative(0.01)), &level, Some(1), scalar)); + assert!(!plans(Some(relative(0.01)), &level, Some(1), doubled)); + assert!(plans(None, &level, Some(1), doubled)); +} + +/// A ratio's linear bound holds while its denominator is off by at most +/// 0.1 (`ErrorContract::linear_bound_holds_up_to_per_input`): a +/// denominator readout at relative 0.2 cannot be read under any target, +/// one at 0.05 can. +#[test] +fn k_caps_a_ratio_s_denominator_at_its_validity_cap() { + let ratio_plans = |denominator_error: f64| { + let mut memo = empty_memo(); + let (p99, _) = ddsketch_readouts(&mut memo, 0.99, &[(0.01, 1.0)]); + let (p50, _) = ddsketch_readouts(&mut memo, 0.5, &[(denominator_error, 1.0)]); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + query(&mut memo, ratio, Some(relative(1.0))); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + match solve(&physical, &every_10s(1), &w, &[], &[]).0 { + Ok(_) => true, + Err(SelectionMilpError::Infeasible) => false, + Err(other) => panic!("{other:?}"), + } + }; + assert!(ratio_plans(0.05)); + assert!(!ratio_plans(0.2)); +} + +/// The same readout every 10 s and every 20 s at relative 0.01 with δ = +/// 0.01 over the whole result, returning 10 values every 10 s and 100 +/// every 20 s: the readout failing with 1e-3 (cost 1) meets 1e-3 per +/// value, not 1e-4, where the one failing with 1e-5 (cost 2) is needed. +/// The contexts differ in what (K) reads ($n_q$), so the 20 s one is not a +/// copy: each selects its own readout. +#[test] +fn contexts_whose_queries_split_their_failure_probability_differently_are_not_copies() { + let mut memo = empty_memo(); + let (readout_group, readouts) = + ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1e-3), (0.01, 1e-5)]); + for values in [10, 100] { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(values), + }); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + // The scan costs 10, so neither readout is dearer than a feasible plan. + let mut w = vec![0.0; physical.physical_candidates.len()]; + w[0] = 10.0; + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 2.0; + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if let asap_types::stages::ExecutionContext::Evaluation { grid } = + selectable.context(selectable.executions()[index].context) + { + if grid.period_ms == 20_000 { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + } + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + let selected = candidates(&solved); + assert!( + selected.contains(&physical_of(&physical, readouts[0])), + "{selected:?}" + ); + assert!( + selected.contains(&physical_of(&physical, readouts[1])), + "{selected:?}" + ); +} + +/// The same readout group as `q` every 10 s and as `scalar(q) * 2` every +/// 20 s, both at relative 0.01, with only a readout at relative 0.01: the +/// operator root accepts it, the scalar expression's operand only exact +/// values (no error contract covers it). The contexts differ in the +/// metrics their outputs allow, so the 20 s one is not a copy whose root +/// rows (K) could leave out: no plan meets both. +#[test] +fn contexts_whose_roots_allow_different_metrics_are_not_copies() { + let mut memo = empty_memo(); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1.0)]); + let doubled = QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::PromqlScalarFromVector(readout_group)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }); + for root in [QueryRoot::Operator(readout_group), doubled] { + memo.add_query(MemoQuery { + root, + accuracy_target: Some(relative(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let (solved, _) = solve(&physical, &[every(10_000), every(20_000)], &w, &[], &[]); + assert_eq!(solved.unwrap_err(), SelectionMilpError::Infeasible); +} + +/// The same readout every 10 s and every 20 s, returning 10 values every +/// 10 s and 100 every 20 s, at relative 0.01 without a failure probability +/// or with δ = 0.01 per value: (K) reads the same root limits for both +/// (no per-value δ, or 0.01 per value whatever $n_q$), so the 20 s context +/// is a copy of the 10 s one. +#[test] +fn contexts_whose_queries_differ_only_in_values_not_split_by_k_are_copies() { + let per_value = relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + for target in [relative(0.01), per_value] { + let mut memo = empty_memo(); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1.0)]); + for values in [10, 100] { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some(target.clone()), + number_of_values_per_query_evaluation: Some(values), + }); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if let asap_types::stages::ExecutionContext::Evaluation { grid } = + selectable.context(selectable.executions()[index].context) + { + if grid.period_ms == 20_000 { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + } + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs); + let shared = copies_in_equivalent_contexts(&inputs, &kept, rules.as_ref()); + assert_eq!( + shared.original_of_query, + vec![QueryId(0), QueryId(0)], + "{target:?}" + ); + } +} + +/// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of +/// `values` values, over readouts at relative 0.01 failing with +/// `failure_probabilities` (cheap at cost 1, dear at cost 3) each: how +/// many readouts of each level the selection reads (cheap, dear) and how +/// many times HiGHS solved the MILP. +fn solve_ratio_over_values( + values: u64, + failure_probabilities: [f64; 2], +) -> Result<(usize, usize, usize), SelectionMilpError> { + let mut memo = empty_memo(); + let levels = [ + (0.01, failure_probabilities[0]), + (0.01, failure_probabilities[1]), + ]; + let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); + let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(ratio), + accuracy_target: Some(relative(0.03).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(values), + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let mut w = vec![0.0; physical.physical_candidates.len()]; + for readouts in [&p99_readouts, &p50_readouts] { + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + } + let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); + let solved = solved?; + let selected = candidates(&solved); + let count = |level: usize| { + [&p99_readouts, &p50_readouts] + .iter() + .filter(|readouts| selected.contains(&physical_of(&physical, readouts[level]))) + .count() + }; + Ok((count(0), count(1), solved.solve_count)) +} + +/// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of 1e4 +/// values, 1e-6 per value, over readouts at relative 0.01 failing with +/// 5.4e-7 (cost 1) or 4e-7 (cost 3) each: two at 5.4e-7 fail together with +/// up to 1.08e-6, above 1e-6 by 8e-8, within HiGHS's default feasibility +/// tolerance 1e-7, so the first solve selects the two cheap readouts. The +/// exact check finds the miss, and the second solve, at tolerance 1e-10, +/// selects one of each (9.4e-7, within the limit). +#[test] +fn a_selection_missing_its_target_at_the_default_tolerance_is_solved_again_at_the_least_tolerance() +{ + assert_eq!( + solve_ratio_over_values(10_000, [5.4e-7, 4e-7]), + Ok((1, 1, 2)) + ); +} + +/// The same ratio over readouts failing with 4e-7 (cost 1) or 3e-7 +/// (cost 3) each: two cheap readouts fail with up to 8e-7, within 1e-6, +/// so the solve at HiGHS's default tolerances meets the target and is +/// the only solve. +#[test] +fn a_selection_meeting_its_target_at_the_default_tolerance_is_solved_once() { + assert_eq!(solve_ratio_over_values(10_000, [4e-7, 3e-7]), Ok((2, 0, 1))); +} + +/// The same ratio over 1e8 values (1e-10 per value) and 1e10 values +/// (1e-12 per value), with readouts failing at 5.4e-11 / 4e-11 and +/// 5.4e-13 / 4e-13: two cheap readouts miss the limit by 8e-12 and 8e-14, +/// within HiGHS's least feasibility tolerance 1e-10, so both solves, at +/// the default tolerances and again at 1e-10, select them. The exact check +/// after the second solve finds the miss, and selection fails rather than +/// return a plan that may not meet the target. +#[test] +fn a_per_value_failure_probability_below_the_solver_precision_is_an_error() { + for (values, failure_probabilities, per_value) in [ + (100_000_000, [5.4e-11, 4e-11], 1e-10), + (10_000_000_000, [5.4e-13, 4e-13], 1e-12), + ] { + let solved = solve_ratio_over_values(values, failure_probabilities); + let Err(SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value: Some(delta), + }) = solved + else { + panic!("{values}: {solved:?}"); + }; + assert_eq!(query, QueryId(0)); + assert!((delta - per_value).abs() <= 1e-6 * per_value, "{delta}"); + } +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 92da7901e..765d3332c 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -14,6 +14,7 @@ use std::collections::{BTreeMap, HashMap}; +use super::accuracy::AccuracyRules; use super::model::{has_one_reader, selection_cost_per_second, SelectionInputs}; use super::COST_RELATIVE_TOLERANCE; use crate::selectable::QueryId; @@ -33,11 +34,15 @@ type TreeInputs = Vec>; /// can take off ($\min(0, c_{\kappa,u,v,i})$ per input) is over the /// bound. Such an execution is in no optimal plan, as long as every /// other selected execution costs at least 0 with its selected inputs. -/// None is ruled out when no feasible plan is found. -pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { +/// None is ruled out when no feasible plan is found, the accuracy +/// constraint (K) (`rules`) included. +pub(crate) fn dearer_than_a_feasible_plan( + inputs: &SelectionInputs<'_>, + rules: Option<&AccuracyRules>, +) -> Vec { let selectable = inputs.selectable; let count = selectable.executions().len(); - let Some(bound) = feasible_plan_cost_per_second(inputs) else { + let Some(bound) = feasible_plan_cost_per_second(inputs, rules) else { return vec![false; count]; }; let costs = inputs.costs; @@ -91,8 +96,15 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec) -> Option { +/// way, or when the plan found misses an accuracy target under (K) +/// (`rules`, [`AccuracyRules::selection_meets_targets`]): the cheapest +/// readouts of a ratio may each meet their level and compose above the +/// target, and a bound from such a plan could leave out the readouts the +/// optimum needs. +fn feasible_plan_cost_per_second( + inputs: &SelectionInputs<'_>, + rules: Option<&AccuracyRules>, +) -> Option { let selectable = inputs.selectable; let executions = selectable.executions(); let count = executions.len(); @@ -135,6 +147,13 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { continue 'repair; } } + if rules.is_some_and(|rules| { + rules + .selection_meets_targets(inputs, &plan.edges, &answering) + .is_err() + }) { + return None; + } let (cost, _) = selection_cost_per_second(inputs, &plan.executions, &plan.edges, &answering); return cost.is_finite().then_some(cost); diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index e999389ad..4234483d9 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -9,7 +9,9 @@ use asap_cost::{summary_type_name, CostModel}; use asap_types::accuracy::AccuracyMetric; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; -use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::plan::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, +}; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidate, PrunedCandidate}; use thiserror::Error; @@ -112,8 +114,14 @@ impl RejectionCause { /// bound that met its target, and the sketches its output reads. #[derive(Debug, Clone, PartialEq)] pub struct AcceptedAccuracy { - /// The metric of the accepted error bound; `None` for an exact output, - /// which meets every metric, and for a query without an accuracy target. + /// Whether the query has an accuracy target. + pub has_accuracy_target: bool, + /// The metric of the accepted error bound, that of the first sketch + /// readout the output reads (in plan order; with a target naming no + /// metric, readouts in several metrics may each be within it, as (K) + /// bounds each metric apart, and this is the first's); `None` for an + /// exact output, which meets every metric, and for a query without an + /// accuracy target. pub accepted_metric: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. @@ -121,9 +129,11 @@ pub struct AcceptedAccuracy { } impl std::fmt::Display for AcceptedAccuracy { - /// E.g. `accepted rank (Kll)`, or `accepted exact`. + /// E.g. `accepted rank (Kll)`, `accepted exact`, or `no accuracy + /// target (DDSketch)`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.accepted_metric { + None if !self.has_accuracy_target => write!(f, "no accuracy target")?, None => write!(f, "accepted exact")?, Some(metric) => { let name = serde_json::to_value(metric) @@ -157,9 +167,14 @@ pub struct Selection { pub milp_variable_count: usize, /// The selection MILP's constraint rows. pub milp_constraint_count: usize, - /// How long building and solving the selection MILP took. + /// How long building and solving the selection MILP took, the solve + /// time summed over `milp_solve_count` solves. pub milp_construction_ms: u64, pub milp_solve_ms: u64, + /// How many times HiGHS solved the MILP: 1, or 2 when the selection at + /// HiGHS's default feasibility tolerances (1e-7) missed an accuracy + /// target when checked exactly and was solved again at 1e-10. + pub milp_solve_count: usize, } /// Why Stage 3 selected no plan. @@ -188,6 +203,18 @@ pub enum ErrorSelectingPlan { /// The memo search's MILP solver failed. #[error("MILP solver: {0}")] Solver(String), + /// Query `query`'s accuracy target is below the solver's precision: + /// the MILP's selection misses it when checked exactly, so no plan is + /// returned rather than one that may not meet the target. Example: a + /// failure probability of 1e-12 per output value (δ = 0.01 over 1e10 + /// values), below the solver's feasibility tolerance 1e-10. + #[error( + "query {query}: an accuracy target is below the solver's precision (failure probability per output value {failure_probability_per_output_value:?}); the solver's plan misses it, so no plan is returned" + )] + AccuracyTargetBelowSolverPrecision { + query: usize, + failure_probability_per_output_value: Option, + }, /// The selection MILP implements cost model 1 only (#1095 §3). #[error("the selection MILP does not support {0}")] UnsupportedCostModel(String), @@ -224,12 +251,16 @@ pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { } /// Per query, the sketches its output reads in `plan` (each sketch build -/// below its root, in node order) and, when it reads one, its target's -/// metric. +/// below its root, in node order) and, when it has an accuracy target and +/// reads a sketch, the metric of the first sketch readout below its root +/// (`CandidateForMemoGroup::required_accuracy`, the requirement (K) +/// checked), whatever metric the target names: a target naming none +/// reports the one it accepted. pub(crate) fn accepted_accuracy( physical: &PhysicalASAPMemoCandidates, plan: &PhysicalASAPDAG, ) -> Vec { + let memo = &physical.logical_candidates.memo; let mut operator_roots = plan.roots.iter(); let mut producers_of: HashMap> = HashMap::new(); for edge in &plan.edges { @@ -238,10 +269,7 @@ pub(crate) fn accepted_accuracy( .or_default() .push(edge.producer); } - physical - .logical_candidates - .memo - .queries() + memo.queries() .iter() .enumerate() .map(|(query, memo_query)| { @@ -253,10 +281,35 @@ pub(crate) fn accepted_accuracy( Some((_, expression)) => expression.operator_refs().into_iter().copied().collect(), None => vec![*operator_roots.next().expect("one operator root per query")], }; - let sketches = sketches_below(plan, &producers_of, starts); + let mut sketches: Vec = Vec::new(); + let mut readout_metrics: Vec = Vec::new(); + for node in nodes_below(plan, &producers_of, starts) { + match &node.payload { + Operator::ASAP( + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, + ) if matches!(summary_type, FieldDataType::Sketch(_)) => { + let name = summary_type_name(summary_type); + if !sketches.contains(&name) { + sketches.push(name); + } + } + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) => { + // The metric of the error the readout's candidate + // was required to meet, the one (K) checked. + readout_metrics.extend(node.selected_physical_candidate.and_then(|v| { + let id = physical.physical_candidates[v.0].logical_candidate_id; + Some(memo.candidate(id).required_accuracy()?.metric) + })); + } + _ => {} + } + } + let has_accuracy_target = memo_query.accuracy_target.is_some(); AcceptedAccuracy { - accepted_metric: (!sketches.is_empty()) - .then(|| memo_query.accuracy_target.as_ref()?.accuracy_metric) + has_accuracy_target, + accepted_metric: (has_accuracy_target && !sketches.is_empty()) + .then(|| readout_metrics.first().copied()) .flatten(), sketches, } @@ -264,13 +317,12 @@ pub(crate) fn accepted_accuracy( .collect() } -/// The summary type names of the sketch builds at or below `starts` in -/// `plan`, in node order, each once. -fn sketches_below( - plan: &PhysicalASAPDAG, +/// The nodes at or below `starts` in `plan`, in node order. +fn nodes_below<'a>( + plan: &'a PhysicalASAPDAG, producers_of: &HashMap>, starts: Vec, -) -> Vec { +) -> impl Iterator { let mut below = BTreeSet::new(); let mut pending = starts; while let Some(id) = pending.pop() { @@ -278,22 +330,9 @@ fn sketches_below( pending.extend(producers_of.get(&id).into_iter().flatten().copied()); } } - let mut sketches: Vec = Vec::new(); - for node in plan.nodes.iter().filter(|node| below.contains(&node.id)) { - if let Operator::ASAP( - ASAPOp::SummaryAgg { summary_type, .. } - | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, - ) = &node.payload - { - if matches!(summary_type, FieldDataType::Sketch(_)) { - let name = summary_type_name(summary_type); - if !sketches.contains(&name) { - sketches.push(name); - } - } - } - } - sketches + plan.nodes + .iter() + .filter(move |node| below.contains(&node.id)) } #[cfg(test)] diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs index 18d908020..5ed6aa715 100644 --- a/crates/types/src/accuracy/target/mod.rs +++ b/crates/types/src/accuracy/target/mod.rs @@ -64,6 +64,18 @@ impl AccuracyMetric { Self::Relative, Self::Rank, ]; + /// How many metrics there are: `ALL`'s length, the size of an array + /// holding one value per metric. + pub const COUNT: usize = Self::ALL.len(); + + /// The position of `self` in [`Self::ALL`], the index of its entry in + /// an array of one value per metric: `Relative` is 3. + pub fn index(self) -> usize { + Self::ALL + .iter() + .position(|&listed| listed == self) + .expect("AccuracyMetric::ALL lists every metric") + } } /// The wire name of an [`AccuracyMetric`]. @@ -285,6 +297,30 @@ impl AccuracyTarget { self.max_error == 0.0 && self.failure_probability == Some(0.0) } + /// The failure probability each output value of one evaluation must + /// meet: the target's δ split evenly, by the union bound, over the + /// values its scope covers when one evaluation returns + /// `values_per_evaluation` ($n_q$; [`AccuracyScope::max_values_covered`]). + /// `None` when the target sets no δ, and when that count is not known + /// for a δ over the whole result. A readout value's failure probability + /// splits δ further over the readout values an output value depends + /// on, and is built on this one. + /// + /// Example: δ = 0.01 over the whole result of a query returning 100 + /// series gives 0.0001 per output value; with the number of series + /// unknown, `None`; per value (`AccuracyScope::PerValue`), 0.01 either + /// way. + pub fn failure_probability_per_output_value( + &self, + values_per_evaluation: Option, + ) -> Option { + let delta = self.failure_probability?; + let values = self + .accuracy_scope + .max_values_covered(values_per_evaluation)?; + Some(delta / values.max(1) as f64) + } + /// The metrics this target accepts an error in: its metric, every /// metric when it names none. pub fn metrics(&self) -> Vec { diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs index d5f690d39..290b10a52 100644 --- a/crates/types/src/accuracy/target/tests.rs +++ b/crates/types/src/accuracy/target/tests.rs @@ -332,3 +332,25 @@ fn exact_targets() { AccuracyScope::WholeResult ); } + +/// A whole-result δ splits over the values one evaluation returns; a +/// per-value δ does not; no δ, or a whole-result one over an unknown count, +/// has none. +#[test] +fn a_whole_result_failure_probability_splits_over_the_output_values() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + assert_eq!( + whole.failure_probability_per_output_value(Some(100)), + Some(1e-4) + ); + assert_eq!(whole.failure_probability_per_output_value(None), None); + let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); + assert_eq!( + per_value.failure_probability_per_output_value(None), + Some(0.01) + ); + assert_eq!( + AccuracyTarget::max_error(0.01).failure_probability_per_output_value(Some(100)), + None + ); +} diff --git a/crates/types/src/stages/logical/candidate/mod.rs b/crates/types/src/stages/logical/candidate/mod.rs index fae0c4d41..a9e646f9a 100644 --- a/crates/types/src/stages/logical/candidate/mod.rs +++ b/crates/types/src/stages/logical/candidate/mod.rs @@ -4,7 +4,7 @@ use crate::ir::operator::asap::summary_type_of_operator; use crate::ir::schema::FieldDataType; use crate::ir::Operator; -use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId}; +use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId, RequiredAccuracy}; use crate::summary_type::{parameter_list, SketchAlgorithm, SketchParams}; /// One candidate: position `index_in_group` in group `group`'s candidates @@ -58,6 +58,19 @@ impl CandidateForMemoGroup { origin, } } + + /// The accuracy a readout requires of the state it reads, which is + /// also the error the readout adds: its input's `required_accuracy`. + /// `None` for any other candidate, and for a readout no query with an + /// accuracy target gives a level. + /// + /// Example: a DDSketch quantile readout Pass 2 gives relative 0.01 at + /// δ = 1e-3 returns that requirement; a scan returns `None`. + pub fn required_accuracy(&self) -> Option { + self.input_requirements + .iter() + .find_map(|requirements| requirements.required_accuracy) + } } /// The θ of each build candidate of `build`: every listed θ diff --git a/crates/types/src/stages/logical/group/column_properties.rs b/crates/types/src/stages/logical/group/column_properties.rs index b3dd1dd6d..30115e8d2 100644 --- a/crates/types/src/stages/logical/group/column_properties.rs +++ b/crates/types/src/stages/logical/group/column_properties.rs @@ -50,6 +50,15 @@ pub enum ValueSign { Unknown, } +impl ValueSign { + /// Whether every value has one sign (`NonNegative` or `NonPositive`): + /// then a relative error ε on each value is at most ε on their sum, and + /// DDSketch's relative bound on a quantile holds. + pub fn is_one_sign(self) -> bool { + matches!(self, Self::NonNegative | Self::NonPositive) + } +} + /// Where an output column's values come from, a group property derived /// from the input groups (concepts doc, section 3, "Column lineage"). It /// names a column across groups where a schema field's name cannot: two diff --git a/crates/types/src/stages/logical/group/mod.rs b/crates/types/src/stages/logical/group/mod.rs index cf087d8ea..9a6e39e03 100644 --- a/crates/types/src/stages/logical/group/mod.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -105,6 +105,21 @@ pub struct GroupProperties { } impl GroupProperties { + /// The `column_signs` entry of a state group's state column (its first + /// field that is not plain): the sign of the values its states were + /// updated with. `None` for a group without a state column. + /// + /// Example: a DDSketch state over `latency`, declared non-negative, + /// is `NonNegative`; over a gauge with no declared sign, `Unknown`. + pub fn state_column_sign(&self) -> Option { + let state = self + .schema + .fields + .iter() + .position(|field| !field.is_plain())?; + self.column_signs.get(state).copied() + } + /// The sketch algorithm of the state the group outputs; `None` for /// rows, values and exact states. pub fn sketch_algorithm_of_state(&self) -> Option { diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 7d52645b2..3637934d5 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -22,7 +22,7 @@ use crate::stages::{ LogicalRequirements, MemoGroup, MemoGroupId, }; use crate::summary_type::SketchStatistic; -use crate::workload::bounds::source_name; +use crate::workload::bounds::{declared_max_distinct_per_evaluation, source_name}; use crate::workload::{series_facts, DataArrival, DataWorkload, SourceFacts}; use hash_consing::{ expression_hash, same_candidate, same_serialized_expression, serialized_expression, @@ -125,6 +125,26 @@ impl Memo { } } + /// The most distinct items one distinct-count summary over `group`'s + /// rows counts in one evaluation: the sum of the bounds the data + /// workload declares for the sources scanned below it + /// (`GroupProperties::scanned_sources`, + /// [`declared_max_distinct_per_evaluation`]); `None` when one of them + /// declares none. + /// + /// Example: `count(distinct user_id)` over `Scan events` with `events` + /// bounded at 9,636 gives 9,636 for every group above the scan. + pub fn max_distinct_per_evaluation(&self, group: MemoGroupId) -> Option { + declared_max_distinct_per_evaluation( + &self.source_facts, + self.groups[group.0 as usize] + .group_properties + .scanned_sources + .iter() + .map(source_name), + ) + } + /// The groups the `by` keys of the state `group` is, or reads, form /// ([`Self::max_values_per_evaluation`]). fn declared_groups_of_state(&self, group: MemoGroupId) -> Option { diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 6d66f4ef1..0a40e109f 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -463,3 +463,39 @@ fn max_values_per_evaluation_bounds_a_readout_by_its_states_keys() { assert_eq!(memo.max_values_per_evaluation(quantile), Some(values)); } } + +/// A distinct count over `events`, bounded at 9,636 distinct items per +/// evaluation, counts at most 9,636 in every group above the scan; a join +/// of two bounded sources counts at most their sum; a source without a +/// bound bounds nothing. +#[test] +fn max_distinct_per_evaluation_sums_the_scanned_sources_bounds() { + let bounded = |bound: Option| crate::workload::SourceFacts { + max_distinct_per_evaluation: bound, + ..Default::default() + }; + let data = crate::workload::DataWorkload { + sources: [ + ("events".to_string(), bounded(Some(9_636))), + ("clicks".to_string(), bounded(Some(364))), + ("logs".to_string(), bounded(None)), + ] + .into(), + ..Default::default() + }; + let mut memo = Memo::new(&data); + let mut scan_of = |source: &str| memo.find_or_add_group(scan(source, &[], vec![])).unwrap(); + let (events, clicks, logs) = (scan_of("events"), scan_of("clicks"), scan_of("logs")); + let state = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Hll), events)) + .unwrap(); + assert_eq!(memo.max_distinct_per_evaluation(events), Some(9_636)); + assert_eq!(memo.max_distinct_per_evaluation(state), Some(9_636)); + assert_eq!(memo.max_distinct_per_evaluation(logs), None); + let both = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Concat { + children: vec![events, clicks], + })) + .unwrap(); + assert_eq!(memo.max_distinct_per_evaluation(both), Some(10_000)); +} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index d5b4227fb..e56309f88 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -67,7 +67,10 @@ pub enum RootPruneReason { /// evaluable at the required failure probability (an HLL distinct /// count, whose failure probability is unknown), or with no guarantee /// at all (UnivMon's distinct count), read by a query with an accuracy - /// target: its error has no bound. + /// target: its error has no bound. Two more cases: a DDSketch quantile + /// over values of unknown sign (its relative guarantee holds only over + /// values of one sign), and any sketch readout under an exact target + /// (δ 0), which no probabilistic guarantee meets. NoGuarantee { consumer: CandidateForMemoGroupId, input: usize, diff --git a/crates/types/src/test_support/memo_builders.rs b/crates/types/src/test_support/memo_builders.rs index 09c5765d6..f9af4dc06 100644 --- a/crates/types/src/test_support/memo_builders.rs +++ b/crates/types/src/test_support/memo_builders.rs @@ -111,6 +111,23 @@ pub fn with_series(metric: &str, series_count: u64) -> DataWorkload { } } +/// A data workload declaring only that `metric`'s `value` is non-negative +/// (`SourceFacts::column_signs`), as a latency is: DDSketch's quantile +/// guarantee holds over it. +pub fn with_non_negative_values(metric: &str) -> DataWorkload { + DataWorkload { + sources: [( + metric.to_string(), + SourceFacts { + column_signs: [("value".to_string(), crate::stages::ValueSign::NonNegative)].into(), + ..SourceFacts::default() + }, + )] + .into(), + ..DataWorkload::default() + } +} + /// The state group of a sketch build of `algorithm` over `child` grouped /// by `reduction`, one build candidate per listed θ /// (`listed_build_parameters`). diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 32bff6d3b..d43517f9c 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -13,7 +13,9 @@ use crate::ir::{ use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; use crate::stages::ValueSign; use crate::summary_type::SketchStatistic; -use crate::workload::{DataWorkload, PrometheusMetricType, SourceFacts, ValueMagnitudeRange}; +use crate::workload::{ + series_facts, DataWorkload, PrometheusMetricType, SourceFacts, ValueMagnitudeRange, +}; /// A scan source's name: its PromQL metric or SQL table. pub fn source_name(source: &Source) -> &str { @@ -474,6 +476,26 @@ pub fn declared_value_magnitude_range<'a>( }) } +/// The most distinct items one distinct-count summary over the sources +/// `scanned` counts per evaluation: the sum of the bound each declares +/// ([`SourceFacts::max_distinct_per_evaluation`]), each looked up as a +/// PromQL selector reads it ([`DataWorkload::series_facts`]: a histogram +/// family `foo`'s bound covers `foo_bucket`, `foo_sum` and `foo_count`, as +/// its `series_count` does). Unlike [`declared_value_magnitude_range`], +/// whose ranges differ between a family's series, a bound on the distinct +/// items counted holds for each series of the family. `None` when one of +/// them declares none, or the sum overflows. E.g. `events` bounded at +/// 9,636 and `clicks` at 1,000 give 10,636. +pub(crate) fn declared_max_distinct_per_evaluation<'a>( + sources: &BTreeMap, + scanned: impl IntoIterator, +) -> Option { + scanned.into_iter().try_fold(0u64, |total, source| { + let (facts, _) = series_facts(sources, source)?; + total.checked_add(facts.max_distinct_per_evaluation?) + }) +} + /// Rows per millisecond the sources in `sources` receive together, each /// at its own rate: `series_count` series each sampled every /// `sample_interval`, or, for a source that does not declare both, its diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index cd05aa860..f23fb5384 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -185,10 +185,10 @@ pub struct SourceFacts { /// count separately: the bound is per output row. The deployment must /// enforce it; an observed distinct count is not a bound. It is for a /// distinct-count guarantee whose failure probability per output row - /// depends on the number of distinct items (classic HLL, a - /// distinct-count UnivMon); no guarantee in `asap_accuracy` reads it - /// yet, so those readouts serve no query with an accuracy target. KMV's - /// guarantee needs no bound. + /// depends on the number of distinct items: classic HLL's reads it (a + /// readout over sources without a bound serves no query with an + /// accuracy target); a distinct-count UnivMon has no guarantee yet. + /// KMV's guarantee needs no bound. #[serde(default)] pub max_distinct_per_evaluation: Option, /// How many distinct values each label (PromQL) or column (SQL) of the diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 079a5581e..c698b99cf 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -73,7 +73,7 @@ over a scope, a per-value failure probability `δ` becomes `V · δ` by the unio bound, without assuming independence, so a sketch shared by several groups (Hydra, a keyed Count-Min) is covered the same way (`AccuracyScope::max_values_covered` gives `V`). A guarantee whose own failure probability is -unknown (generic HLL) serves no query with an accuracy target +unknown (HLL without a distinct bound) serves no query with an accuracy target (`RootPruneReason::NoGuarantee`), and a top-k readout is admitted on its heap size and its value guarantee alone (the contracts table). @@ -182,8 +182,8 @@ The built-in models currently include: |---|---| | Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require error propagation | | KLL | Normalized rank error from the empirical 99% calibration `2.296 / k^0.9723`, fitted on Apache DataSketches' C++ KLL (`a9b42755072b`); below a failure probability of 0.01 that error is scaled by the Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`, a model rather than a measurement, and the guarantee names that contract apart | -| DDSketch | Relative value error from alpha under the supported estimator/domain contract; operators sensitive to sign or range require additional domain evidence | -| Generic HLL | RSE magnitude with unknown failure probability; not a general confidence theorem | +| DDSketch | Relative value error from alpha under the supported estimator/domain contract; a quantile's bound holds only over values of one sign (the state column's `column_signs` non-negative or non-positive, "Sign conditions"), else its failure probability is unknown and the readout serves no query with an accuracy target | +| Classic HLL | Relative error certified at the required failure probability over a source declaring at most `max_distinct_per_evaluation` distinct items (summed over the sources scanned below the state), from the linear-counting collision bound under independent uniform bucket hashes; without such a bound, or above the contract's range (`m/2` items at precision `p`), only the RSE magnitude with unknown failure probability | | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | | KMV | Provable distinct-count bound from the `k` smallest hashes | @@ -264,6 +264,30 @@ them, and sharing does not justify reducing the failure budget. Per-result guarantees also do not imply a simultaneous guarantee across every query or evaluation time in a dashboard. +### Sign conditions + +Two rules hold only over values of one sign. Both read the group property +`column_signs` (declared per source column in `SourceFacts::column_signs`, a +counter's value non-negative, and derived through each operator: a negation +flips it, `abs` makes it non-negative) and test it with +`ValueSign::is_one_sign`, non-negative or non-positive: + +- DDSketch's relative bound on a quantile. The state column's sign + (`GroupProperties::state_column_sign`, the sign of the values the states + were updated with) must be one sign; otherwise the guarantee + (`guarantee/ddsketch.rs`) keeps its metric, relative error, with an unknown + failure probability, so the readout serves no query with an accuracy target + (`RootPruneReason::NoGuarantee`) and `theoretical_guarantee` returns none. + `quantile_over_time(0.99, latency[1h])` with `latency` declared + non-negative holds at α; over a gauge of unknown sign it does not. +- A sum's or an average's relative error (the contracts table): the input + column's sign must be one sign, as `|Σ (x̂_i − x_i)| ≤ ε Σ |x_i| = + ε |Σ x_i|` holds then, while values of unknown sign may cancel. + +The cost model reads whether a DDSketch state's values are non-negative from +the sources' declared signs instead, so it can disagree with this rule where +an operator changes the sign (DDSketch over `-latency`). + ## Readout requirements for composed queries Pass 2 gives each readout the accuracy it is required to meet. A query's @@ -340,6 +364,108 @@ An exact target gets no approximate level. A model with only a fixed 1% failure contract cannot meet a level that needs a smaller failure probability; another supported configuration or candidate is needed. +## Checking each query's target in selection: constraint (K) + +Pass 2 gives each readout requirement levels, and the logical-edge check keeps +the producers that meet them. A composed query reads several readouts, each +at one of its levels, and only the selection decides which: Stage 3's +selection MILP checks that the readouts a selection combines compose within +each query's target. This is constraint (K). + +**Variables.** Per execution `c` (a candidate in one evaluation context) +that a query with a target reaches, and that is its own original (not in a +copy of an equivalent context): its error bound `E_{c,m}` per metric `m` and +its failure probability `D_c`, continuous; per input `i`, `F_{c,i,m}` and +`H_{c,i}`, what it reads from the producer selected on `i`. + +**Rows.** With `y_{u,c,i}` the edge variables and `z_{q,v}` the choice of +execution `v` answering query `q`: + +- `E_{c,m} ≥ ε_c[m = m_c] + Σ_i β_{c,i,m} F_{c,i,m}`, with + `F_{c,i,m} ≥ E_{u,m} − M_{u,m}(1 − y_{u,c,i})` per selectable producer `u`; + `ε_c` is a readout's own error, its required accuracy + (`CandidateForMemoGroup::required_accuracy`), 0 for any other candidate. +- The metric rule on the selected path: `E_{u,m} ≤ M_{u,m}(1 − y_{u,c,i})` + when `c` has no contract in `m`. An input with `β = 0` passes none. +- The validity cap: `F_{c,i,m} ≤ b_{c,i}` where the contract's linear bound + holds only up to `b` (a ratio's denominator). +- `D_c ≥ δ_c + Σ_i H_{c,i}`, `H_{c,i} ≥ D_u − M_u(1 − y_{u,c,i})`: a union + bound over the producers selected. +- At each output of a query with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` + in each metric the root allows (the target's at an operator root, none for + the operands of a scalar expression no contract covers, 0 elsewhere), and + `D_v ≤ δ_q / V + M(1 − z_{q,v})` with `V` the values the target's scope + covers (`AccuracyTarget::failure_probability_per_output_value`, `n_q` for a + whole result); no `D` row without a δ. Each limit is widened as the edge + check widens it (`tolerated_limit`). + +`D` adds each selected producer's δ once, not once per readout value an +output value depends on: `max(q)` over 1,000 series reads 1,000 readout +values. That is sound because Pass 2's levels already split the query's δ +over the readout values its output can depend on, so each readout's δ is the +query's over 1,000. + +**Constant errors.** A readout's state is exact, so its error is its own +requirement under every selection: it has no `E` or `D` variable, and its +consumers read `Σ_u ε_u y_{u,c,i}` (at most one `y` of an input is 1). This is +exactly the variable form with `E_u = ε_u`, and it removes the big-M rows of +every readout. + +**Ceilings.** `M` is the largest error any selection gives an execution, +computed in one pass in the memo's topological order (every selectable edge +reads a group ordered before the consumer's), each input taking its largest +producer's ceiling, by the same composition rule as the rows. The rule is +written once (`AccuracyRules::compose`), and also checks a whole selection +(`AccuracyRules::selection_meets_targets`): the greedy upper bound that prunes +executions dearer than a feasible plan uses its plan only when it meets (K), +since the cheapest readouts of a ratio may each meet their level and compose +above the target. + +**Solver tolerance.** HiGHS's feasibility tolerances are absolute, 1e-7 by +default, while a per-value δ is often far smaller: δ = 0.01 over a whole +result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth: two +readouts failing with 5.4e-7 each compose to 1.08e-6 and pass that limit +within the default tolerance. So HiGHS first solves at its defaults, and the +selection is then checked exactly (`AccuracyRules::selection_meets_targets`, +in f64). A selection that misses a target is solved once more at primal and +MIP feasibility tolerance 1e-10, the least HiGHS accepts, and checked again. +A row may still pass its limit by up to 1e-10, which for a per-value δ near +or below 1e-10 can be orders of magnitude: at δ = 1e-12 per value (δ = 0.01 +over 1e10 values) the solver may select readouts failing with 1.08e-12 in +all. A selection that misses a target after the second solve is an error +naming the query and its per-value δ ("an accuracy target is below the +solver's precision"); planning never returns a plan that may not meet its +target. The stricter tolerance is paid for only when the check needs it: +solving every (K) at 1e-10 made RangeHeavy-8users about 6× slower, deriving +the tolerance from the smallest limit solved RollupHeavy-8users about 2.8× +slower, and scaling (K)'s rows by their limits solved RangeHeavy-8users +about 4× slower. + +**Context copies.** A copy of an equivalent evaluation context shares its +original's variables and has no rows of its own. Two contexts are copies only +when each output's root limits (the tolerated error per metric and failure +probability per output value) are equal, so the copy's rows would repeat the +original's: queries at δ = 0.01 per value returning 10 and 100 values are +copies; over a whole result they are not. + +### Example: allocation in a ratio + +A is `quantile_over_time(0.95, latency[1h])` at rank error 0.01, and B is +`quantile_over_time(0.99, latency[1h]) / quantile_over_time(0.5, latency[1h])` +at relative error 0.03, over non-negative `latency`. A reads a KLL readout at +rank 0.01. B's ratio passes each operand's relative error on with factor +`1/(1 − 0.1) = 1.111`, so Pass 2 gives each DDSketch readout below it the +levels of the grid up to its cap; with levels 0.02 (cost 1) and 0.01 (cost 3): + +| p99 readout | p50 readout | B's composed error | meets 0.03 | +|---|---|---|---| +| 0.02 | 0.02 | 1.111 · 0.04 = 0.044 | no | +| 0.02 | 0.01 | 1.111 · 0.03 = 0.033 | no | +| 0.01 | 0.01 | 1.111 · 0.02 = 0.022 | yes | + +(K) selects both 0.01 readouts, cost 6. A greedy plan reading the two +cheapest readouts (cost 2) misses B's target, so it bounds nothing. + ## Evidence and trust boundaries `AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the @@ -362,14 +488,19 @@ an observation into a guarantee. ### Example: HLL distinct counts -HLL's guarantee is its relative standard error, `1.04 / √m` for `m` buckets, -with an unknown failure probability. Its readout's metric is relative error, -but no failure probability can be evaluated, so a distinct count read from -HLL serves no query with an accuracy target -(`RootPruneReason::NoGuarantee`); KMV's distinct count, whose bound does not -depend on the number of distinct items, serves such a query. Missing evidence -leaves generic HLL confidence unknown. Invalid or infeasible contracts cannot -authorize the result. +HLL's generic guarantee is its relative standard error, `1.04 / √m` for `m` +buckets, with an unknown failure probability. Its readout's metric is relative +error, but no failure probability can be evaluated, so without more a distinct +count read from HLL serves no query with an accuracy target +(`RootPruneReason::NoGuarantee`). When the data workload bounds the distinct +items one evaluation counts (`SourceFacts::max_distinct_per_evaluation`, +summed over the sources scanned below the state), classic HLL's +linear-counting branch is certified: `COUNT(DISTINCT user_id)` within relative +0.05 at δ = 0.01 holds at precision 18 over at most 9,636 distinct users, and +over 1,000,000 (above the contract's `m/2`) it does not. KMV's distinct count, +whose bound does not depend on the number of distinct items, serves such a +query without a bound. Missing evidence leaves generic HLL confidence unknown. +Invalid or infeasible contracts cannot authorize the result. ## Sharing across consumers @@ -443,14 +574,16 @@ accuracy/src/ │ # the error contracts, each readout's required-accuracy │ # levels; MAX_ERROR_LEVELS ├── logical_edge/ # ErrorMetricsOfMemo::edge_meets_logical_requirements -│ # (α^log); within_limit, the requirement comparison +│ # (α^log); within_limit and tolerated_limit, the +│ # requirement comparison └── guarantee/ ├── mod.rs # theoretical_guarantee (one lookup at a failure │ # probability per algorithm) ├── binomial.rs # binomial_upper_tail ├── kll/ - ├── ddsketch.rs - ├── hll.rs # HLL (unknown failure probability) + ├── ddsketch.rs # relative error α; its sign condition + ├── hll/ # HLL: classic certificate over a distinct bound, else + │ # RSE (unknown failure probability) ├── cms/ ├── count_sketch/ ├── cardinality/ # KMV model diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 6e18189ca..7d3e11dac 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -202,7 +202,7 @@ asap_stage3_plan_selection::select_plan( | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory}` chooses among the candidates that fit | | `PlanOutput.plan` | The selected plan, one `PhysicalASAPDAG`: `roots` answer the operator queries and `scalar_roots` the standalone scalar queries, in `QueryWorkload::entries()` order; each materialized node carries its `retention_ms`. Its cost is `asap_cost::plan_cost` of it | -| `PlanningRun` | `pass1_memo`, `logical_asap_candidates`, `physical_asap_candidates`, `selection` (the plan and Stage 3's `Selection`: `accepted_accuracy_per_query`, `pruned_candidates`, `milp_variables`, `milp_constraints`, `milp_construction_ms`, `milp_solve_ms`) and `report` (Pass 2's and Stage 2's counts and each stage's time); `pass1_memo` is kept only with `PlanSearchSettings::inspect` | +| `PlanningRun` | `pass1_memo`, `logical_asap_candidates`, `physical_asap_candidates`, `selection` (the plan and Stage 3's `Selection`: `accepted_accuracy_per_query`, `pruned_candidates`, `milp_variables`, `milp_constraints`, `milp_construction_ms`, `milp_solve_ms`, `milp_solve_count`) and `report` (Pass 2's and Stage 2's counts and each stage's time); `pass1_memo` is kept only with `PlanSearchSettings::inspect` | Each candidate's `ResourceUsage` holds its usage without prices: `cpu_ops_per_second`, `scanned_bytes_per_second`, `average_memory_bytes`,