From e65767acb72d9cebb3ebb47f98bc1b3ef61e2885 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:44:24 +0000 Subject: [PATCH 1/4] refactor(stage3): constraint (K) per execution and metric; sign-conditioned DDSketch guarantee, HLL distinct count from the source's distinct bound (#1207 PR 8) Constraint (K) in Stage 3's selection MILP: per execution and error metric, E and D composed along the selected edges through each candidate's error contract (with its validity caps and the metric rule), and checked at each query root in the metrics the root allows, with the target's failure probability per value. A copy of an equivalent context has no rows of its own; copies pair only queries whose (K) rows are equal (target, n_q and the metrics each output allows). The greedy upper bound is used only when its plan meets (K). Accuracy: DDSketch's quantile guarantee holds only over values of one sign; classic HLL's distinct count is certified over the scanned sources' declared distinct bound. Types: AccuracyTarget::failure_probability_per_value, GroupProperties::state_column_sign, Memo::max_distinct_per_evaluation. Report: the accepted metric is the readout's; a query without a target reports none. --- crates/accuracy/src/error_contract/mod.rs | 2 +- crates/accuracy/src/guarantee/cms/tests.rs | 5 +- .../src/guarantee/count_sketch/tests.rs | 1 + crates/accuracy/src/guarantee/hll.rs | 20 - crates/accuracy/src/guarantee/hll/mod.rs | 165 +++++ crates/accuracy/src/guarantee/hll/tests.rs | 96 +++ crates/accuracy/src/guarantee/mod.rs | 37 +- crates/accuracy/src/guarantee/tests.rs | 15 +- crates/accuracy/src/lib.rs | 9 +- crates/accuracy/src/logical_edge/mod.rs | 77 +- crates/accuracy/src/logical_edge/tests.rs | 137 +++- crates/accuracy/src/metric_rule/mod.rs | 4 +- crates/accuracy/src/readout/mod.rs | 4 +- .../tests/accuracy_at_query_roots.rs | 174 +++++ crates/integration-tests/tests/main.rs | 1 + .../tests/pass2_promql/accuracy_and_grids.rs | 7 +- .../tests/pass2_promql/exact_only_groups.rs | 16 +- .../tests/planning_common/mod.rs | 7 +- .../src/tests/dead_candidates.rs | 6 +- .../src/tests/exact_only_groups.rs | 5 +- crates/stage3-plan-selection/src/lib.rs | 8 +- .../src/milp/accuracy.rs | 682 +++++++++++++++++- .../src/milp/equivalent_contexts.rs | 46 +- .../src/milp/model/mod.rs | 45 +- .../src/milp/model/tests.rs | 367 +++++++++- .../src/milp/upper_bound.rs | 27 +- crates/stage3-plan-selection/src/report.rs | 97 ++- crates/types/src/accuracy/target/mod.rs | 20 + crates/types/src/accuracy/target/tests.rs | 15 + crates/types/src/stages/logical/group/mod.rs | 15 + crates/types/src/stages/logical/memo/mod.rs | 20 + crates/types/src/stages/logical/memo/tests.rs | 36 + .../types/src/test_support/memo_builders.rs | 17 + crates/types/src/workload/data.rs | 8 +- docs/design_docs/concepts/accuracy-models.md | 34 +- 35 files changed, 2025 insertions(+), 200 deletions(-) delete mode 100644 crates/accuracy/src/guarantee/hll.rs create mode 100644 crates/accuracy/src/guarantee/hll/mod.rs create mode 100644 crates/accuracy/src/guarantee/hll/tests.rs create mode 100644 crates/integration-tests/tests/accuracy_at_query_roots.rs diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index 97b16bf63..ba76a5a56 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -419,7 +419,7 @@ impl ErrorContract { /// Example: `count` outputs how many values it reads, whatever their /// error ($\beta = 0$): false; `sum` passes its input's relative error /// on ($\beta = 1$): true. - pub(crate) fn input_error_reaches_output(&self, input: usize) -> bool { + pub fn input_error_reaches_output(&self, input: usize) -> bool { self.error_factor_per_input[input] > 0.0 } } diff --git a/crates/accuracy/src/guarantee/cms/tests.rs b/crates/accuracy/src/guarantee/cms/tests.rs index 12b35605f..32379b26f 100644 --- a/crates/accuracy/src/guarantee/cms/tests.rs +++ b/crates/accuracy/src/guarantee/cms/tests.rs @@ -7,8 +7,9 @@ fn heap_evaluation_retains_frequency_metric() { depth: 5, heap_size: 10, }; - let topk_frequency = guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0) - .expect("heap sketch still provides per-key frequency intervals"); + let topk_frequency = + guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0, None) + .expect("heap sketch still provides per-key frequency intervals"); assert_eq!( topk_frequency.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1 diff --git a/crates/accuracy/src/guarantee/count_sketch/tests.rs b/crates/accuracy/src/guarantee/count_sketch/tests.rs index f0505631f..877015098 100644 --- a/crates/accuracy/src/guarantee/count_sketch/tests.rs +++ b/crates/accuracy/src/guarantee/count_sketch/tests.rs @@ -36,6 +36,7 @@ fn count_sketch_uses_an_l2_guarantee() { value: Some("1".into()), }, 1.0, + None, ) .expect("CountSketch has a parameter-derived L2 guarantee"); assert_eq!( diff --git a/crates/accuracy/src/guarantee/hll.rs b/crates/accuracy/src/guarantee/hll.rs deleted file mode 100644 index ef83b3e67..000000000 --- a/crates/accuracy/src/guarantee/hll.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! HLL's relative standard error, without a failure probability. - -use super::*; - -pub(super) fn guarantee(params: &SketchParams) -> Option { - let SketchParams::Hll { precision, .. } = params else { - return None; - }; - Some(super::bounded_guarantee( - AccuracyMetric::Relative, - relative_standard_error(*precision), - ProbabilityExpr::Unknown { - statistic: "hll_estimator_failure_probability".into(), - }, - )) -} -/// HLL's relative standard error, `1.04 / √m` for `m = 2^precision` buckets. -fn relative_standard_error(precision: u8) -> f64 { - 1.04 / 2f64.powi(i32::from(precision)).sqrt() -} diff --git a/crates/accuracy/src/guarantee/hll/mod.rs b/crates/accuracy/src/guarantee/hll/mod.rs new file mode 100644 index 000000000..62db84dc7 --- /dev/null +++ b/crates/accuracy/src/guarantee/hll/mod.rs @@ -0,0 +1,165 @@ +//! HLL's distinct count: its relative standard error, without a failure +//! probability, and classic HLL's certified guarantee over a source of at +//! most a declared number of distinct items per evaluation +//! ([`classic_guarantee_at`]), from its linear-counting branch. +//! +//! The certificate assumes independent uniform bucket hashes and an upper +//! bound on the distinct items in the complete evaluation population +//! (including all merged tumbling windows), which the data workload declares +//! (`SourceFacts::max_distinct_per_evaluation`); observations alone do not +//! establish it. It is not an RSE-to-normal conversion or a fit. + +use super::*; + +/// HLL's guarantee at `params` for its distinct count: classic HLL's +/// certified one at `failure_probability` when the source declares at most +/// `max_distinct_per_evaluation` distinct items and the contract covers that +/// bound ([`classic_guarantee_at`]); else the relative standard error, whose +/// failure probability is unknown. +pub(super) fn guarantee( + params: &SketchParams, + failure_probability: f64, + max_distinct_per_evaluation: Option, +) -> Option { + let SketchParams::Hll { precision, .. } = params else { + return None; + }; + max_distinct_per_evaluation + .and_then(|bound| classic_guarantee_at(*precision, failure_probability, bound)) + .or_else(|| { + Some(super::bounded_guarantee( + AccuracyMetric::Relative, + relative_standard_error(*precision), + ProbabilityExpr::Unknown { + statistic: "hll_estimator_failure_probability".into(), + }, + )) + }) +} + +/// HLL's relative standard error, `1.04 / √m` for `m = 2^precision` buckets. +fn relative_standard_error(precision: u8) -> f64 { + 1.04 / 2f64.powi(i32::from(precision)).sqrt() +} + +/// Classic HLL's guarantee at `precision` over a source of at most +/// `max_distinct` distinct items per evaluation: the least relative error +/// whose failure probability ([`ClassicHllConfidence`]) is at most +/// `failure_probability`, with that failure probability. `None` when no +/// relative error below 1 is certified (the bound is outside the contract +/// for this precision). +/// +/// Example: at precision 18 and δ = 0.01, a bound of 9,636 certifies a +/// relative error just under 0.05; a bound of 1,000,000 certifies none. +fn classic_guarantee_at( + precision: u8, + failure_probability: f64, + max_distinct: u64, +) -> Option { + let max_distinct = u32::try_from(max_distinct).ok()?; + let certified = |relative_error: f64| { + ClassicHllConfidence::new(max_distinct, relative_error) + .and_then(|model| model.failure_probability(precision)) + .is_some_and(|delta| delta <= failure_probability) + }; + // The failure probability falls as the relative error grows: bisect + // for the least certified one, keeping the certified end. + let (mut uncertified, mut certified_error) = (1e-6, 1.0 - 1e-6); + if !certified(certified_error) { + return None; + } + for _ in 0..60 { + let middle = (uncertified + certified_error) / 2.0; + match certified(middle) { + true => certified_error = middle, + false => uncertified = middle, + } + } + let delta = + ClassicHllConfidence::new(max_distinct, certified_error)?.failure_probability(precision)?; + Some(super::bounded_guarantee( + AccuracyMetric::Relative, + certified_error, + ProbabilityExpr::Constant { value: delta }, + )) +} + +/// The largest precision classic HLL's contract covers. +const MAX_PRECISION: u8 = 18; + +/// The largest distinct bound the contract supports at any precision: +/// `m/2` at precision 18. Evaluating a bound is O(bound). +const MAX_CONTRACT_DISTINCT: u32 = 1 << 17; + +/// A finite-population contract for `m * ln(m / zero_registers)` with the +/// classic HLL small-range switch, over at most `max_distinct` distinct +/// items and at relative error `relative_error`. Hashing is assumed +/// independent and uniform. +#[derive(Debug, Clone, Copy)] +struct ClassicHllConfidence { + max_distinct: u32, + relative_error: f64, +} + +impl ClassicHllConfidence { + /// `None` for a bound of 0 or above [`MAX_CONTRACT_DISTINCT`], or a + /// relative error outside [1e-6, 1). + fn new(max_distinct: u32, relative_error: f64) -> Option { + (max_distinct > 0 + && max_distinct <= MAX_CONTRACT_DISTINCT + && relative_error.is_finite() + && (1e-6..1.0).contains(&relative_error)) + .then_some(Self { + max_distinct, + relative_error, + }) + } + + /// The probability that the estimate at `precision` is off by more + /// than the relative error, for any population up to the bound; `None` + /// outside the contract (a precision outside 4..=18, a bound above + /// `m/2`, or a relative error the deterministic overestimate exceeds). + /// + /// A finite bound, not an asymptotic RSE fit. With N distinct hashes + /// and K occupied buckets, C = N − K collision arrivals satisfy + /// P(C ≥ t) ≤ λ^t / t!, λ = N(N−1)/(2m): each arrival's conditional + /// collision probability is at most (i−1)/m, and a union bound over t + /// arrivals is bounded by the t-th power of their sum divided by t!. + /// + /// N ≤ m/2 makes the classic raw estimate ≤ 2 α_m m < 2.5 m, so the + /// small-range switch always uses L = −m ln(1 − K/m). Then + /// K ≤ L ≤ N + N²/(2(m − N)). The latter bounds overestimation + /// deterministically; underestimation implies C > ε N. The collision + /// bound is maximized over every integer N in the contract, not just + /// its upper end (small populations' tails matter). + fn failure_probability(&self, precision: u8) -> Option { + if !(4..=MAX_PRECISION).contains(&precision) { + return None; + } + let m = f64::from(1u32 << precision); + let max_n = f64::from(self.max_distinct); + // Reserve numerical slack; do not certify sub-floating-point error. + let eps = self.relative_error - 1e-8; + if max_n > m / 2.0 || max_n / (2.0 * (m - max_n)) > eps { + return None; + } + let mut log_factorial = vec![0.0; self.max_distinct as usize + 1]; + for i in 1..log_factorial.len() { + log_factorial[i] = log_factorial[i - 1] + (i as f64).ln(); + } + let mut worst = 0.0_f64; + for n in 2..=self.max_distinct { + let nf = f64::from(n); + // Including a boundary collision event is conservative. + let t = ((eps * nf).floor() as usize + 1).min(n as usize); + let lambda = nf * (nf - 1.0) / (2.0 * m); + let log_tail = (t as f64) * lambda.ln() - log_factorial[t]; + worst = worst.max(log_tail.min(0.0).exp()); + } + // Never return a spurious zero from underflow or cancellation. + Some((worst * (1.0 + 1e-10) + 1e-12).min(1.0)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/accuracy/src/guarantee/hll/tests.rs b/crates/accuracy/src/guarantee/hll/tests.rs new file mode 100644 index 000000000..6f60bd8a4 --- /dev/null +++ b/crates/accuracy/src/guarantee/hll/tests.rs @@ -0,0 +1,96 @@ +use super::*; + +/// The failure probability at a relative error, over a bound, at a +/// precision; 1 outside the contract. +fn classic_failure_probability(precision: u8, relative_error: f64, max_distinct: u64) -> f64 { + u32::try_from(max_distinct) + .ok() + .and_then(|n| ClassicHllConfidence::new(n, relative_error)) + .and_then(|model| model.failure_probability(precision)) + .unwrap_or(1.0) +} + +/// The largest distinct bound certified at δ = 0.01 for a target ε, by +/// precision; a bound outside the contract certifies nothing. +#[test] +fn largest_certified_bounds_by_precision() { + for (eps, precision, largest) in [ + (0.05, 15, 1176), + (0.05, 18, 9636), + (0.02, 17, 1842), + (0.02, 18, 3791), + (0.01, 18, 72), + ] { + assert!( + classic_failure_probability(precision, eps, largest) <= 0.01, + "eps={eps} p={precision} n={largest}" + ); + assert!( + classic_failure_probability(precision, eps, largest + 1) > 0.01, + "eps={eps} p={precision} n={}", + largest + 1 + ); + } + for too_large in [u64::from(MAX_CONTRACT_DISTINCT) + 1, u64::MAX] { + assert_eq!(classic_failure_probability(18, 0.05, too_large), 1.0); + } +} + +/// Exact occupancy probabilities independently check both tails for every +/// population up to the bound. +#[test] +fn probability_bound_dominates_exact_occupancy_distribution() { + for precision in 4..=10 { + let m = 1usize << precision; + let max_n = 64.min(m / 2); + for eps in [0.05, 0.2, 0.6] { + let model = ClassicHllConfidence::new(max_n as u32, eps).unwrap(); + let Some(bound) = model.failure_probability(precision) else { + continue; + }; + let mut occupancy = vec![0.0; max_n + 1]; + occupancy[0] = 1.0; + for n in 1..=max_n { + let mut next = vec![0.0; max_n + 1]; + for k in 0..n { + next[k] += occupancy[k] * k as f64 / m as f64; + next[k + 1] += occupancy[k] * (m - k) as f64 / m as f64; + } + occupancy = next; + let actual: f64 = occupancy + .iter() + .enumerate() + .filter_map(|(k, &prob)| { + let estimate = -(m as f64) * (-(k as f64) / (m as f64)).ln_1p(); + ((estimate - n as f64).abs() > eps * n as f64).then_some(prob) + }) + .sum(); + assert!( + actual <= bound + 1e-12, + "p={precision} n={n} eps={eps}: {actual}>{bound}" + ); + } + } + } +} + +/// `COUNT(DISTINCT user_id)` within relative 0.05 at δ = 0.01 from classic +/// HLL at precision 18 is certified over a source of at most 9,636 +/// distinct users per evaluation; over 1,000,000 (outside the contract) and +/// without a bound only the relative standard error is stated, whose +/// failure probability is unknown. +#[test] +fn a_distinct_count_is_certified_over_the_source_s_distinct_bound() { + let precision_18 = SketchParams::Hll { precision: 18 }; + let at = |bound| guarantee(&precision_18, 0.01, bound).unwrap(); + let certified = at(Some(9_636)); + assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::Relative); + let error = certified.spec.error.evaluate().unwrap(); + assert!(error <= 0.05 && error > 0.04, "{error}"); + assert!(certified.spec.failure_probability.evaluate().unwrap() <= 0.01); + for uncertified in [Some(1_000_000), None] { + let guarantee = at(uncertified); + assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); + assert_eq!(guarantee.spec.failure_probability.evaluate(), None); + } +} diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 0227d0008..0bd1fae3b 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -23,21 +23,29 @@ use binomial::binomial_upper_tail; /// `failure_probability`; where structural parameters set them apart (a /// Count-Min's width and depth, HLL's precision), the one θ fixes, whose /// failure probability may exceed `failure_probability` or be unknown -/// (HLL's). A Hydra state reads each subpopulation's state, so it has -/// that state's guarantee. +/// (HLL's, unless the source declares its distinct bound +/// `max_distinct_per_evaluation`: classic HLL's is then certified at +/// `failure_probability`). A Hydra state reads each subpopulation's +/// state, so it has that state's guarantee. fn guarantee_of_parameters( params: &SketchParams, query: &SketchStatistic, failure_probability: f64, + max_distinct_per_evaluation: Option, ) -> Option { match params { SketchParams::Kll { .. } => kll::guarantee(params, failure_probability), SketchParams::Kmv { .. } => cardinality::guarantee(params, failure_probability), - SketchParams::Hydra(hydra) => { - guarantee_of_parameters(&hydra.per_subpopulation(), query, failure_probability) - } + SketchParams::Hydra(hydra) => guarantee_of_parameters( + &hydra.per_subpopulation(), + query, + failure_probability, + max_distinct_per_evaluation, + ), SketchParams::DDSketch { .. } => ddsketch::guarantee(params), - SketchParams::Hll { .. } => hll::guarantee(params), + SketchParams::Hll { .. } => { + hll::guarantee(params, failure_probability, max_distinct_per_evaluation) + } SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => cms::guarantee(params), SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { count_sketch::guarantee(params) @@ -50,18 +58,26 @@ fn guarantee_of_parameters( /// fails at most `failure_probability` rarely: its metric, and whether its /// failure probability is known at all. `None` when the summary type does /// not answer `statistic` or has no error model for it. +/// `max_distinct_per_evaluation` is the source's declared distinct bound +/// (`Memo::max_distinct_per_evaluation`), which a distinct count reads. pub(crate) fn guarantee_before_failure_probability_check( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, failure_probability: f64, + max_distinct_per_evaluation: Option, ) -> Option { if !matches!(summary_type, FieldDataType::Sketch(_)) || !SummaryTypeCapability::of(summary_type).answers(statistic) { return None; } - guarantee_of_parameters(parameters, statistic, failure_probability) + guarantee_of_parameters( + parameters, + statistic, + failure_probability, + max_distinct_per_evaluation, + ) } /// The guarantee of an error at most `bound` in `metric`, failing with @@ -89,18 +105,23 @@ fn bounded_guarantee( /// that rarely. `None` when this θ cannot fail that rarely, when the /// summary type does not answer `statistic` (its capability row) or has no /// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch -/// readouts without a certified bound). +/// readouts without a certified bound). A distinct count from classic HLL +/// is certified only over a source of at most `max_distinct_per_evaluation` +/// distinct items (`Memo::max_distinct_per_evaluation`); `None` without that +/// bound. pub fn theoretical_guarantee( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, failure_probability: f64, + max_distinct_per_evaluation: Option, ) -> Option { guarantee_before_failure_probability_check( summary_type, parameters, statistic, failure_probability, + max_distinct_per_evaluation, ) .filter(|guarantee| holds_at_failure_probability(guarantee, failure_probability)) } diff --git a/crates/accuracy/src/guarantee/tests.rs b/crates/accuracy/src/guarantee/tests.rs index f53608525..ea98d238e 100644 --- a/crates/accuracy/src/guarantee/tests.rs +++ b/crates/accuracy/src/guarantee/tests.rs @@ -49,8 +49,9 @@ fn every_listed_parameters_has_a_guarantee_at_one_percent() { let errors: Vec = parameter_list(algorithm) .iter() .map(|parameters| { - let guarantee = theoretical_guarantee(&summary_type, parameters, &statistic, 0.01) - .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); + let guarantee = + theoretical_guarantee(&summary_type, parameters, &statistic, 0.01, None) + .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); assert_eq!(guarantee.spec.accuracy_metric, metric, "{algorithm:?}"); guarantee.spec.error.evaluate().unwrap() }) @@ -77,7 +78,7 @@ fn a_guarantee_is_evaluated_at_the_required_failure_probability() { let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); let k = SketchParams::Kll { k: 547 }; let error_at = |delta| { - theoretical_guarantee(&kll, &k, &quantile, delta) + theoretical_guarantee(&kll, &k, &quantile, delta, None) .unwrap() .spec .error @@ -92,9 +93,9 @@ fn a_guarantee_is_evaluated_at_the_required_failure_probability() { value: None, }; let depth = |depth| SketchParams::Cms { width: 272, depth }; - assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2).is_some()); - assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4).is_none()); - assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2, None).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4, None).is_none()); + assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4, None).is_some()); // A statistic the summary type does not answer has no guarantee. - assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01).is_none()); + assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01, None).is_none()); } diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index abf154e7d..f335e77c3 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -5,7 +5,9 @@ //! ([`AccuracyEvidence`], in `evidence`), each operator's error contract //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the -//! metric of a readout's error from `readout`), each readout's required +//! metrics each query root allows, [`query_roots_with_target`], and the +//! metric of a readout's error, [`readout_error_metric`], from `readout`), +//! each readout's required //! accuracy ([`readout_levels`], in `readout_caps`), and $\alpha^{\log}$ of //! a logical edge ([`ErrorMetricsOfMemo::edge_meets_logical_requirements`], //! in `logical_edge`, with a bound's comparison to a limit @@ -26,6 +28,7 @@ mod readout_caps; pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; -pub use logical_edge::within_limit; -pub use metric_rule::ErrorMetricsOfMemo; +pub use logical_edge::{tolerated_limit, within_limit}; +pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; +pub use readout::readout_error_metric; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/logical_edge/mod.rs b/crates/accuracy/src/logical_edge/mod.rs index 56542b462..beb50cc51 100644 --- a/crates/accuracy/src/logical_edge/mod.rs +++ b/crates/accuracy/src/logical_edge/mod.rs @@ -4,9 +4,12 @@ //! ([`within_limit`]). use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; -use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; -use asap_types::summary_type::{MergedError, SketchParams, SketchStatistic, SummaryTypeCapability}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason, ValueSign}; +use asap_types::summary_type::{ + MergedError, SketchAlgorithm, SketchParams, SketchStatistic, SummaryTypeCapability, +}; use crate::guarantee::{guarantee_before_failure_probability_check, holds_at_failure_probability}; use crate::readout::sketch_readout; @@ -21,7 +24,16 @@ const SATISFACTION_TOLERANCE: f64 = 1e-9; /// `limit`: at most `limit` widened by [`SATISFACTION_TOLERANCE`], so a /// bound equal to it up to floating-point noise meets it. pub fn within_limit(evaluated: f64, limit: f64) -> bool { - evaluated <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON + evaluated <= tolerated_limit(limit) +} + +/// The largest value [`within_limit`] admits for `limit`: `limit` widened +/// by [`SATISFACTION_TOLERANCE`]. Stage 3's root checks of constraint (K) +/// bound a composed error by it, so they accept what an edge accepts. +/// +/// Example: a limit of 0.01 admits up to 0.01 · (1 + 1e-9) + 2.2e-16. +pub fn tolerated_limit(limit: f64) -> f64 { + limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON } impl ErrorMetricsOfMemo { @@ -37,7 +49,11 @@ impl ErrorMetricsOfMemo { /// guarantee of the producer's θ for the readout's statistic, at the /// required failure probability, is within the required error /// ([`within_limit`]) and in its metric: the deployment's measurement - /// (`evidence`) when it has one, else the theoretical guarantee; a + /// (`evidence`) when it has one, else the theoretical guarantee when + /// its sign condition holds for the state's values + /// ([`sign_condition_holds`]), a distinct count's over the scanned + /// sources' declared distinct bound + /// (`Memo::max_distinct_per_evaluation`); a /// merged state (any producer but a build) must keep the guarantee /// (`MergedError::Exact` or `AsIfBuiltOverUnion`). A top-k's heap /// holds its `k` whatever the requirement. @@ -68,7 +84,8 @@ impl ErrorMetricsOfMemo { }) } }; - let Operator::ASAP(ASAPOp::SummaryEstimate { .. }) = consumer_candidate.expression.operator + let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = + consumer_candidate.expression.operator else { let Some(producer_metric) = self.readout_error_metric(producer) else { return Ok(()); @@ -101,18 +118,25 @@ impl ErrorMetricsOfMemo { let measured = evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); let allowed = self.error_metrics_allowed_on_input_edge(consumer, input); + // The sign of the values the read states were updated with. + let state_sign = memo.groups()[summary_input.0 as usize] + .group_properties + .state_column_sign(); // The theoretical guarantee before its failure-probability check, // computed once for both checks below. - let theoretical = (measured.is_none() && (allowed.is_some() || required.is_some())) - .then(|| { - guarantee_before_failure_probability_check( - summary_type, - parameters, - statistic, - failure_probability, - ) - }) - .flatten(); + let theoretical = (measured.is_none() + && (allowed.is_some() || required.is_some()) + && sign_condition_holds(summary_type, statistic, state_sign)) + .then(|| { + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + failure_probability, + memo.max_distinct_per_evaluation(summary_input), + ) + }) + .flatten(); if let Some(allowed) = allowed { // A query with a target reads this readout: its error needs a // metric and a failure probability a guarantee states. @@ -166,6 +190,29 @@ impl ErrorMetricsOfMemo { } } +/// Whether the theoretical guarantee of reading `statistic` from a state +/// of `summary_type` holds for states updated with values of sign `sign` +/// (`GroupProperties::state_column_sign`; concepts doc, section 2, "Sign +/// conditions"): DDSketch's relative error on a quantile needs values of +/// one sign, `NonNegative` or `NonPositive`; every other guarantee holds +/// whatever their sign. +/// +/// Example: `quantile_over_time(0.99, latency[1h])` from DDSketch with +/// `latency` declared non-negative holds; over a gauge of unknown sign it +/// does not, as the sketch's buckets around 0 have no relative bound. +fn sign_condition_holds( + summary_type: &FieldDataType, + statistic: &SketchStatistic, + sign: Option, +) -> bool { + match (summary_type, statistic) { + (FieldDataType::Sketch(SketchAlgorithm::DDSketch), SketchStatistic::Quantile { .. }) => { + matches!(sign, Some(ValueSign::NonNegative | ValueSign::NonPositive)) + } + _ => true, + } +} + /// A heap variant's heap size. fn heap_size(parameters: &SketchParams) -> Option { match parameters { diff --git a/crates/accuracy/src/logical_edge/tests.rs b/crates/accuracy/src/logical_edge/tests.rs index 40d51baf9..bf44659a7 100644 --- a/crates/accuracy/src/logical_edge/tests.rs +++ b/crates/accuracy/src/logical_edge/tests.rs @@ -4,7 +4,9 @@ use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; -use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; +use asap_types::stages::{ + CandidateForMemoGroup, LogicalRequirements, OriginOfCandidateForMemoGroup, RequiredAccuracy, +}; use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_builders::query; use asap_types::test_support::memo_fixtures::scan; @@ -145,3 +147,136 @@ fn a_heap_smaller_than_k_fails_the_topk_readouts_edge() { } assert_eq!(small, 2 * 28); } + +/// The edge from the build of `algorithm` at `parameters` over `Scan events` (whose +/// facts are `facts`) into a readout of `statistic` required at +/// `required`, which a query with `target` reads. +fn readout_edge( + facts: asap_types::workload::SourceFacts, + algorithm: SketchAlgorithm, + parameters: SketchParams, + statistic: SketchStatistic, + required: RequiredAccuracy, + target: AccuracyTarget, +) -> Result<(), RootPruneReason> { + let data = DataWorkload { + sources: [("events".to_string(), facts)].into(), + ..DataWorkload::default() + }; + let mut memo = Memo::new(&data); + let events = memo.find_or_add_group(scan("events", &[], vec![])).unwrap(); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: events, + summary_type: FieldDataType::Sketch(algorithm), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let state = memo.find_or_add_group(build.clone()).unwrap(); + let producer = memo + .add_candidate( + state, + CandidateForMemoGroup::new( + build, + Some(parameters), + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) + .unwrap(); + let readout = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: statistic, + }); + let group = memo.find_or_add_group(readout.clone()).unwrap(); + let mut candidate = CandidateForMemoGroup::new( + readout, + None, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ); + candidate.input_requirements = vec![LogicalRequirements { + required_accuracy: Some(required), + }]; + let consumer = memo.add_candidate(group, candidate).unwrap(); + query(&mut memo, group, Some(target)); + let order = memo.topological_order().expect("acyclic"); + ErrorMetricsOfMemo::of(&memo, &order).edge_meets_logical_requirements( + &memo, + producer, + consumer, + 0, + &NoMeasuredGuarantees, + ) +} + +/// `COUNT(DISTINCT user_id)` within relative 0.05 at δ = 0.01 from classic +/// HLL at precision 18: the edge holds over a source declaring at most +/// 9,636 distinct items per evaluation, and fails with `NoGuarantee` over +/// 1,000,000 (outside the contract) or without a bound. +#[test] +fn a_distinct_count_readout_reads_the_scanned_sources_distinct_bound() { + for (bound, holds) in [(Some(9_636), true), (Some(1_000_000), false), (None, false)] { + let facts = asap_types::workload::SourceFacts { + max_distinct_per_evaluation: bound, + ..Default::default() + }; + let edge = readout_edge( + facts, + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 18 }, + SketchStatistic::Cardinality, + RequiredAccuracy { + metric: AccuracyMetric::Relative, + max_error: 0.05, + failure_probability: 0.01, + }, + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ); + match holds { + true => assert!(edge.is_ok(), "{bound:?}: {edge:?}"), + false => assert!( + matches!(edge, Err(RootPruneReason::NoGuarantee { input: 0, .. })), + "{bound:?}: {edge:?}" + ), + } + } +} + +/// DDSketch's relative error on a quantile holds for values of one sign: +/// `quantile_over_time(0.5, events[..])` at relative 0.01 from α = 0.01 +/// holds when `events`' value is declared non-negative, and fails with +/// `NoGuarantee` when its sign is unknown. +#[test] +fn a_ddsketch_quantile_holds_only_over_values_of_one_sign() { + for (sign, holds) in [(Some(ValueSign::NonNegative), true), (None, false)] { + let facts = asap_types::workload::SourceFacts { + column_signs: sign + .map(|sign| ("value".to_string(), sign)) + .into_iter() + .collect(), + ..Default::default() + }; + let edge = readout_edge( + facts, + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + SketchStatistic::Quantile { q: 0.5 }, + RequiredAccuracy { + metric: AccuracyMetric::Relative, + max_error: 0.01, + failure_probability: 1.0, + }, + AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative), + ); + match holds { + true => assert!(edge.is_ok(), "{sign:?}: {edge:?}"), + false => assert!( + matches!(edge, Err(RootPruneReason::NoGuarantee { input: 0, .. })), + "{sign:?}: {edge:?}" + ), + } + } +} diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 9d15c30be..f176c2229 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -285,7 +285,9 @@ fn metric_index(metric: AccuracyMetric) -> usize { /// (which return the value they read) allow the target's metrics /// (`AccuracyTarget::metrics`); any other scalar expression's operands /// allow none, as no error contract covers it, so they are read exactly. -pub(crate) fn query_roots_with_target( +/// Pass 2's readout caps and Stage 3's root checks of constraint (K) read +/// it. +pub fn query_roots_with_target( memo: &Memo, ) -> Vec<(usize, MemoGroupId, &AccuracyTarget, Vec)> { let mut roots = Vec::new(); diff --git a/crates/accuracy/src/readout/mod.rs b/crates/accuracy/src/readout/mod.rs index fc4c9b8bc..4ebc1d69b 100644 --- a/crates/accuracy/src/readout/mod.rs +++ b/crates/accuracy/src/readout/mod.rs @@ -15,7 +15,7 @@ use crate::guarantee::guarantee_before_failure_probability_check; /// error although its failure probability is unknown). `None` for an exact /// state and for a summary type with no listed θ or no guarantee for the /// statistic (UnivMon's distinct count). -pub(crate) fn readout_error_metric( +pub fn readout_error_metric( summary_type: &FieldDataType, statistic: &SketchStatistic, ) -> Option { @@ -24,7 +24,7 @@ pub(crate) fn readout_error_metric( }; let parameters = parameter_list(*algorithm).first()?; Some( - guarantee_before_failure_probability_check(summary_type, parameters, statistic, 1.0)? + guarantee_before_failure_probability_check(summary_type, parameters, statistic, 1.0, None)? .spec .accuracy_metric, ) diff --git a/crates/integration-tests/tests/accuracy_at_query_roots.rs b/crates/integration-tests/tests/accuracy_at_query_roots.rs new file mode 100644 index 000000000..2316e3c1f --- /dev/null +++ b/crates/integration-tests/tests/accuracy_at_query_roots.rs @@ -0,0 +1,174 @@ +//! Constraint (K), each query's composed accuracy at its root, through +//! every planning stage on #1207's problem example (concepts doc, +//! section 1): A, the p95 of `latency` at rank error 0.01, and B, the +//! ratio of its p99 and p50 at relative error 0.03. + +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; +use asap_planner::{DeploymentCapabilities, PlanningModels}; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::summary_type::SketchParams; +use asap_types::workload::PlanningWorkload; + +/// PromQL `queries`, each `(query, accuracy JSON)` every 10 s, over +/// `latency` (1,000 series, one sample per second, ingested, values +/// declared non-negative). +fn workload(queries: &[(&str, &str)]) -> PlanningWorkload { + let entries: Vec = queries + .iter() + .map(|(query, accuracy)| { + format!( + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {accuracy}}}, "predictability": {{"predictable": {{"known_at": null}}}}}}"# + ) + }) + .collect(); + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, + "sources": {{"latency": {{"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000, + "column_signs": {{"value": "non_negative"}}}}}}}}}}"#, + entries.join(",") + )) + .unwrap() +} + +/// The plan Stage 3 selects for `queries`, no raw data kept (every query +/// reads summaries built at ingestion), with accuracy enforced or +/// ignored. +fn plan(queries: &[(&str, &str)], ignore_accuracy: bool) -> PhysicalASAPDAG { + selection(queries, ignore_accuracy).0 +} + +/// [`plan`], with the accepted accuracy Stage 3 reports per query. +fn selection(queries: &[(&str, &str)], ignore_accuracy: bool) -> (PhysicalASAPDAG, Vec) { + let workload = workload(queries); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap(); + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..Default::default() + }; + let models = PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_ignore_accuracy_guarantees(ignore_accuracy); + let roots = lower_promql_roots(&workload).unwrap(); + let run = run_planning_stages_over(roots, &entries, &data, models).unwrap(); + let (plan, selection) = run.run.selection.as_ref().expect("a plan is selected"); + let accepted = selection + .accepted_accuracy_per_query + .iter() + .map(ToString::to_string) + .collect(); + (plan.clone(), accepted) +} + +/// #1207's problem example. +const PROBLEM_EXAMPLE: [(&str, &str); 2] = [ + ( + "quantile_over_time(0.95, latency[1h])", + r#"{"accuracy_metric": "rank", "max_error": 0.01}"#, + ), + ( + "quantile_over_time(0.99, latency[1h]) / quantile_over_time(0.5, latency[1h])", + r#"{"accuracy_metric": "relative", "max_error": 0.03}"#, + ), +]; + +/// The θ of the plan's sketch state nodes. +fn sketch_parameters(plan: &PhysicalASAPDAG) -> Vec { + plan.nodes + .iter() + .filter_map(|node| node.parameters.clone()) + .collect() +} + +/// A meets rank error 0.01 only with a KLL of k ≥ 269 (a DDSketch readout +/// bounds a relative error, not a rank); B reads DDSketch states whose +/// readouts' composed bound $(\alpha_{p99} + \alpha_{p50}) / (1 - 0.1)$ is +/// within 0.03, which two α = 0.02 readouts (each within 0.03 alone) are +/// not. +#[test] +fn each_query_meets_its_target_at_its_root() { + let (plan, accepted) = selection(&PROBLEM_EXAMPLE, false); + assert_eq!( + accepted, + ["accepted rank (Kll)", "accepted relative (DDSketch)"], + "the metric of each query's readouts" + ); + let parameters = sketch_parameters(&plan); + assert!( + parameters + .iter() + .any(|p| matches!(p, SketchParams::Kll { k } if *k >= 269)), + "A reads a KLL meeting rank 0.01: {parameters:?}" + ); + assert!( + !parameters + .iter() + .any(|p| matches!(p, SketchParams::Kll { k } if *k < 269)), + "{parameters:?}" + ); + let mut alphas: Vec = parameters + .iter() + .filter_map(|p| match p { + SketchParams::DDSketch { alpha } => Some(*alpha), + _ => None, + }) + .collect(); + alphas.sort_by(f64::total_cmp); + let (Some(&smallest), Some(&largest)) = (alphas.first(), alphas.last()) else { + panic!("B reads DDSketch: {parameters:?}"); + }; + assert!( + (smallest + largest) / (1.0 - 0.1) <= 0.03 * (1.0 + 1e-9), + "B's composed bound: {alphas:?}" + ); +} + +/// Ignoring accuracy, no query has a target: (K) has no rows and any +/// readout serves, so one summary type answers A's p95 and B's p99 and p50 +/// (which one is a matter of cost), where enforcing accuracy needs a KLL +/// for A's rank target and DDSketch for B's relative one. The report says +/// there is no target to accept (it said "accepted exact (…)"). +#[test] +fn ignoring_accuracy_plans_without_k() { + // The number of sketch algorithms the plan builds. + let algorithms = |plan: &PhysicalASAPDAG| { + sketch_parameters(plan) + .iter() + .map(std::mem::discriminant) + .collect::>() + .len() + }; + assert_eq!(algorithms(&plan(&PROBLEM_EXAMPLE, false)), 2); + let (ignored, accepted) = selection(&PROBLEM_EXAMPLE, true); + assert_eq!(algorithms(&ignored), 1, "{:?}", sketch_parameters(&ignored)); + assert!( + accepted + .iter() + .all(|accepted| accepted.starts_with("no accuracy target (")), + "{accepted:?}" + ); +} + +/// A whole-result failure probability of 0.01 over 1,000 series is 1e-5 +/// per value: a KLL read at that failure probability meets rank error +/// 0.01 only from k = 547 up (the concepts doc's KLL table). +#[test] +fn a_whole_result_failure_probability_is_divided_over_the_values() { + let plan = plan( + &[( + "quantile_over_time(0.95, latency[1h])", + r#"{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}"#, + )], + false, + ); + let parameters = sketch_parameters(&plan); + assert!(!parameters.is_empty()); + for parameters in parameters { + assert!( + matches!(parameters, SketchParams::Kll { k } if k >= 547), + "{parameters:?}" + ); + } +} diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index 9ee627c25..b650d32e6 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -6,6 +6,7 @@ mod physical_common; mod planner_layering_common; mod planning_common; +mod accuracy_at_query_roots; mod aggregate; mod binary_op; mod crate_graph; diff --git a/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs b/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs index 77f0afff8..c84f116b1 100644 --- a/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs +++ b/crates/integration-tests/tests/pass2_promql/accuracy_and_grids.rs @@ -47,7 +47,8 @@ fn live_sketch_readouts(logical: &LogicalASAPMemoCandidates) -> usize { /// `promql_drop_metric_name`. Its contract judges only the value: `abs` /// and the negation keep an absolute error, scaling by a constant a /// relative one, so a sketch readout below each still serves the query, as -/// it does for the bare readout. +/// it does for the bare readout (DDSketch's quantile over `latency`, whose +/// values are declared non-negative). #[test] fn a_function_of_the_value_keeps_the_readouts_below_it() { let absolute = json!({"max_error": 0.05, "accuracy_metric": "absolute"}); @@ -56,8 +57,8 @@ fn a_function_of_the_value_keeps_the_readouts_below_it() { ("count_over_time(m[1m])", &absolute), ("abs(count_over_time(m[1m]))", &absolute), ("-count_over_time(m[1m])", &absolute), - ("quantile_over_time(0.9, m[5m]) / 2", &relative), - ("quantile_over_time(0.9, m[5m]) * 2", &relative), + ("quantile_over_time(0.9, latency[5m]) / 2", &relative), + ("quantile_over_time(0.9, latency[5m]) * 2", &relative), ] { let logical = pass2(query, accuracy.clone()); assert!(live_sketch_readouts(&logical) > 0, "{query}"); diff --git a/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs b/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs index 127f8687d..fb3c67dfa 100644 --- a/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs +++ b/crates/integration-tests/tests/pass2_promql/exact_only_groups.rs @@ -44,14 +44,14 @@ fn live(logical: &LogicalASAPMemoCandidates, group: MemoGroupId) -> bool { }) } -/// `quantile_over_time(0.9, m[5m])` twice, Q0 at rank error 0.01 and Q1 at -/// relative error 0.05: both share the quantile group and both the KLL -/// (rank) and DDSketch (relative) readouts survive, but Q0 reads only the -/// KLL readout and Q1 only the DDSketch one; each reads the other's -/// readout groups only exactly. +/// `quantile_over_time(0.9, latency[5m])` twice (values non-negative), Q0 +/// at rank error 0.01 and Q1 at relative error 0.05: both share the +/// quantile group and both the KLL (rank) and DDSketch (relative) readouts +/// survive, but Q0 reads only the KLL readout and Q1 only the DDSketch +/// one; each reads the other's readout groups only exactly. #[test] fn each_query_reads_only_the_readouts_in_its_targets_metric() { - let query = "quantile_over_time(0.9, m[5m])"; + let query = "quantile_over_time(0.9, latency[5m])"; let logical = promql_pass2(&[ (query, json!({"max_error": 0.01, "accuracy_metric": "rank"})), ( @@ -84,7 +84,7 @@ fn each_query_reads_only_the_readouts_in_its_targets_metric() { } } -/// `count(quantile_over_time(0.9, m[5m]))` at relative error 0.05: the +/// `count(quantile_over_time(0.9, latency[5m]))` at relative error 0.05: the /// count reads the quantiles through an exact count, whose contract /// passes every metric, while Pass 1's Count-Min and UnivMon builds over /// the quantiles (no contract) are dead. A dead candidate makes no group @@ -92,7 +92,7 @@ fn each_query_reads_only_the_readouts_in_its_targets_metric() { #[test] fn a_dead_candidate_makes_no_group_exact_only() { let logical = promql_pass2(&[( - "count(quantile_over_time(0.9, m[5m]))", + "count(quantile_over_time(0.9, latency[5m]))", json!({"max_error": 0.05, "failure_probability": 0.01, "accuracy_metric": "relative"}), )]); let memo = &logical.memo; diff --git a/crates/integration-tests/tests/planning_common/mod.rs b/crates/integration-tests/tests/planning_common/mod.rs index fe0ae4df0..e962f2d52 100644 --- a/crates/integration-tests/tests/planning_common/mod.rs +++ b/crates/integration-tests/tests/planning_common/mod.rs @@ -85,8 +85,9 @@ pub fn selected_plan(w: &FuzzWorkload, settings: &CostInputs) -> PhysicalASAPDAG } /// Pass 2's output for PromQL `queries`, each with its accuracy target, -/// repeating every minute over the gauge `m` and the counter `c`, 1000 -/// series each. +/// repeating every minute over the gauge `m`, the counter `c` and the +/// gauge `latency` whose values are declared non-negative (so DDSketch's +/// quantile guarantee holds over it), 1000 series each. pub fn promql_pass2(queries: &[(&str, Value)]) -> LogicalASAPMemoCandidates { pass2_of( json!("promql"), @@ -94,6 +95,8 @@ pub fn promql_pass2(queries: &[(&str, Value)]) -> LogicalASAPMemoCandidates { json!({ "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + "latency": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000, + "column_signs": {"value": "non_negative"}}, }), None, ) diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs index 158e83b7c..3a24e3c29 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs @@ -8,7 +8,6 @@ use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; use asap_types::summary_type::parameter_list; use asap_types::summary_type::{SketchAlgorithm, SketchParams}; -use asap_types::workload::DataWorkload; use crate::test_fixtures::*; use asap_types::test_support::memo_builders::*; @@ -21,10 +20,11 @@ use asap_types::test_support::memo_fixtures::{arithmetic, range}; /// pruned by the metric rule and every KLL build through it. The readouts /// of DDSketch get the levels 0.02 … 0.001 (cap 0.045), so the builds with /// alpha 0.1 and 0.05 miss every level they could serve. Chain groups are -/// added only for the surviving builds. +/// added only for the surviving builds. `m`'s values are declared +/// non-negative, so DDSketch's quantile guarantee holds. #[test] fn dead_candidates_are_pruned_and_listed_before_chain_groups() { - let mut memo = Memo::new(&DataWorkload::default()); + let mut memo = Memo::new(&with_non_negative_values("m")); let input = scan_group(&mut memo, "m", &["job"], vec![]); let window = group(&mut memo, range(300_000, input)); let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs index 4e9827a4a..a4cbbe366 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs @@ -5,7 +5,6 @@ use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; use asap_types::stages::{ColumnLineage, Memo}; use asap_types::summary_type::SketchAlgorithm; -use asap_types::workload::DataWorkload; use crate::test_fixtures::*; use asap_types::test_support::memo_builders::*; @@ -18,9 +17,11 @@ use asap_types::test_support::memo_fixtures::{arithmetic, range}; /// division carries relative error); Q1's comparison has no contract, so /// Q1 reads the ratio and every group below it only exactly. (Both read /// the rows below a sketch build exactly: a build has no contract.) +/// `latency` is declared non-negative, so DDSketch's quantile guarantee +/// holds. #[test] fn a_query_comparing_a_shared_ratio_reads_it_only_exactly() { - let mut memo = Memo::new(&DataWorkload::default()); + let mut memo = Memo::new(&with_non_negative_values("latency")); let input = scan_group(&mut memo, "latency", &["job"], vec![]); let window = group(&mut memo, range(300_000, input)); let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 96d153d83..87d77d5a8 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -17,9 +17,11 @@ //! exact candidates only, $a_{q,e}$ marking what its selected path reads. It //! builds the plan (`PhysicalASAPDAG`) from the selected executions and //! edges as its last step ([`plan`], each node retaining its candidate's -//! $h_v$, the needs from [`retention`]). Accuracy at each query root, -//! constraint (K), with per-candidate error bounds $E_c$ and $D_c$ -//! (section 4), is the remaining extension. Costs come from `asap-cost`. +//! $h_v$, the needs from [`retention`]). Accuracy at each query root is +//! constraint (K) (section 4): error bounds $E_{\kappa,c,m}$ per execution +//! and metric and failure probabilities $D_{\kappa,c}$, composed along the +//! selected edges and checked at each query's root. Costs come from +//! `asap-cost`. mod cost; mod milp; diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index 43a76e733..6d41ba838 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -1,41 +1,659 @@ //! The accuracy extension of the selection MILP //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 4): its variables and constraint (K), `todo!()` until (K) is -//! added. +//! section 4, "Query composition and root checks"): constraint (K), per +//! execution and per error metric, over the error each execution adds and +//! passes on ([`AccuracyRules`], [`ErrorModel`]), and the same rule +//! checked on one selection ([`AccuracyRules::selection_meets_targets`]), +//! which the greedy upper bound needs to be a feasible plan of (K). -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; -use good_lp::{Constraint, Variable}; +use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; +use asap_accuracy::{query_roots_with_target, tolerated_limit, ErrorContract, ErrorMetricsOfMemo}; use asap_types::accuracy::AccuracyMetric; -use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId}; - -use super::model::SelectionVariables; - -/// The accuracy variables of constraint (K) (section 4), all continuous. -#[allow(dead_code)] // Read once constraint (K) is added. -struct AccuracyVariables { - /// $E_{c,m}$: candidate $c$'s error bound in metric $m$. - error_bound: HashMap<(PhysicalCandidateId, AccuracyMetric), Variable>, - /// $F_{c,i,m}$: the chosen producer's error bound in metric $m$ on input - /// $i$ of $c$. - error_bound_of_input: HashMap<(PhysicalCandidateId, usize, AccuracyMetric), Variable>, - /// $D_c$, by [`PhysicalCandidateId`]. - failure_probability: Vec, - /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. - failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, -} - -/// (K): $E_{c,m}$ per candidate and metric and $D_c$ per candidate, big-M on -/// selected edges, the metric rule on the selected path, and the root -/// checks $E_{v,m_q} \le \varepsilon_q + M_q(1 - z_{q,v})$, -/// $D_v \le \delta_q + (1 - z_{q,v})$. -#[allow(dead_code)] // Called once constraint (K) is added. -fn accuracy_constraints( - physical: &PhysicalASAPMemoCandidates, - selection: &SelectionVariables, +use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId, RequiredAccuracy}; + +use super::equivalent_contexts::SharedVariables; +use super::model::SelectionInputs; +use crate::selectable::QueryId; + +/// The number of error metrics, `AccuracyMetric::ALL`'s length. +const METRIC_COUNT: usize = AccuracyMetric::ALL.len(); + +/// What (K) reads of the memo, built once per selection when some query +/// has an accuracy target: each candidate's error contracts, which +/// executions a query with a target reaches, and what each query root +/// allows. +pub(super) struct AccuracyRules { + /// The candidates' error contracts per metric + /// (`ErrorMetricsOfMemo::error_contract`). + error_metrics: ErrorMetricsOfMemo, + /// Per execution, by position in `SelectableExecutions::executions`, + /// its logical candidate. + candidate_of_execution: Vec, + /// Per execution, whether a query with an accuracy target reaches its + /// group. + targeted: Vec, + /// Per (query, output group its root reads), what its root check + /// bounds the output by. + roots: HashMap<(usize, MemoGroupId), RootLimits>, +} + +/// What a root check of (K) bounds one query output by. +struct RootLimits { + /// The target's ε. + max_error: f64, + /// The metrics allowed at the root ([`query_roots_with_target`]). + allowed_metrics: Vec, + /// $\delta_q$ per value (`AccuracyTarget::failure_probability_per_value`); + /// `None` without a δ, or with one over the whole result and $n_q$ + /// unknown (Pass 2 gives that query's readouts no level). + failure_probability_per_value: Option, +} + +impl AccuracyRules { + /// The rules over `inputs`; `None` when no query has an accuracy + /// target, so (K) has no rows. + pub(super) fn of(inputs: &SelectionInputs<'_>) -> Option { + let physical = inputs.physical; + let memo = &physical.logical_candidates.memo; + if memo + .queries() + .iter() + .all(|query| query.accuracy_target.is_none()) + { + return None; + } + // Pass 2 rejects a cyclic memo (`Pass2RuleError::CyclicDependencies`). + let order = memo + .topological_order() + .expect("Stage 3 reads Pass 2's acyclic memo"); + let candidate_of_execution: Vec = inputs + .selectable + .executions() + .iter() + .map(|execution| { + physical.physical_candidates[execution.candidate.0].logical_candidate_id + }) + .collect(); + let targeted = candidate_of_execution + .iter() + .map(|id| { + memo.groups()[id.group.0 as usize] + .reached_by_queries() + .iter() + .any(|&query| memo.queries()[query].accuracy_target.is_some()) + }) + .collect(); + let roots = query_roots_with_target(memo) + .into_iter() + .map(|(query, root, target, allowed_metrics)| { + let failure_probability_per_value = + target + .failure_probability + .and(target.failure_probability_per_value( + memo.queries()[query].number_of_values_per_query_evaluation, + )); + let limits = RootLimits { + max_error: target.max_error, + allowed_metrics, + failure_probability_per_value, + }; + ((query, root), limits) + }) + .collect(); + Some(Self { + error_metrics: ErrorMetricsOfMemo::of(memo, &order), + candidate_of_execution, + targeted, + roots, + }) + } + + /// Execution `c`'s error contract in metric `m` (by position in + /// `AccuracyMetric::ALL`); `None` where it has none (the metric rule). + fn contract(&self, c: usize, m: usize) -> Option<&ErrorContract> { + self.error_metrics + .error_contract(self.candidate_of_execution[c], AccuracyMetric::ALL[m]) + } + + /// Whether the selection `edges` (by position in + /// `SelectableExecutions::edges`) with the outputs `answering` (query, + /// execution answering it, in `SelectableExecutions::answering` order) + /// meets (K): the composed error of each execution a query with a + /// target reaches, by the rule of [`accuracy_constraints`] on the + /// selected edges, passes its consumers' metric rule and validity caps + /// and, at each output of a query with a target, the root's limits. + /// + /// Example: p99 / p50 at relative 0.03 over two readouts at 0.02 does + /// not meet it (1.111 · 0.04 = 0.044); over two at 0.01 it does. + pub(super) fn selection_meets_targets( + &self, + inputs: &SelectionInputs<'_>, + edges: &[usize], + answering: &[(QueryId, usize)], + ) -> bool { + let selectable = inputs.selectable; + let count = selectable.executions().len(); + let mut reads: Vec> = vec![Vec::new(); count]; + for &position in edges { + let edge = selectable.edges()[position]; + let input = inputs.physical.edge(edge.edge).input_index_of_consumer; + reads[edge.consumer].push((input, edge.producer)); + } + // Composed (E per metric, D) per execution, producers first. + let mut composed: Vec> = vec![None; count]; + let mut on_walk = vec![false; count]; + let mut pending: Vec<(usize, bool)> = answering.iter().map(|&(_, v)| (v, false)).collect(); + while let Some((c, producers_done)) = pending.pop() { + if composed[c].is_some() { + continue; + } + if !producers_done { + if on_walk[c] { + // A cycle, which (E) forbids. + return false; + } + on_walk[c] = true; + pending.push((c, true)); + pending.extend( + reads[c] + .iter() + .filter(|&&(_, u)| composed[u].is_none()) + .map(|&(_, u)| (u, false)), + ); + continue; + } + if !self.targeted[c] { + // No query with a target reaches it: (K) has no rows for + // it, nor for its producers. + composed[c] = Some(([0.0; METRIC_COUNT], 0.0)); + continue; + } + let (mut error_bound, mut failure_probability) = own_error_of(own_error(inputs, c)); + for &(i, u) in &reads[c] { + let Some((producer_error, producer_failure)) = composed[u] else { + return false; + }; + failure_probability += producer_failure; + for (m, &error) in producer_error.iter().enumerate() { + if error <= 0.0 { + continue; + } + let Some(contract) = self.contract(c, m) else { + return false; + }; + if !contract.input_error_reaches_output(i) { + continue; + } + if contract.linear_bound_holds_up_to_per_input[i] + .is_some_and(|holds_up_to| error > tolerated_limit(holds_up_to)) + { + return false; + } + error_bound[m] += contract.error_factor_per_input[i] * error; + } + } + composed[c] = Some((error_bound, failure_probability)); + } + let memo = &inputs.physical.logical_candidates.memo; + for (&(query, group), &(_, v)) in memo.output_groups().iter().zip(answering) { + let Some(limits) = self.roots.get(&(query, group)) else { + continue; + }; + let (error_bound, failure_probability) = + composed[v].expect("each answering execution is composed"); + for (m, &error) in error_bound.iter().enumerate() { + let limit = match limits.allowed_metrics.contains(&AccuracyMetric::ALL[m]) { + true => tolerated_limit(limits.max_error), + false => 0.0, + }; + if error > limit { + return false; + } + } + if limits + .failure_probability_per_value + .is_some_and(|limit| failure_probability > tolerated_limit(limit)) + { + return false; + } + } + true + } +} + +/// $(\varepsilon_c, \delta_c)$ in metric $m_c$ of the execution at +/// position `index`: a readout's requirement, the error it adds; `None` for +/// any other candidate. +fn own_error(inputs: &SelectionInputs<'_>, index: usize) -> Option { + let physical = inputs.physical; + let execution = inputs.selectable.executions()[index]; + let id = physical.physical_candidates[execution.candidate.0].logical_candidate_id; + physical + .logical_candidates + .memo + .candidate(id) + .input_requirements + .iter() + .find_map(|requirements| requirements.required_accuracy) +} + +/// `own` as an error per metric (by position in `AccuracyMetric::ALL`) +/// and a failure probability; 0 when `None`. +fn own_error_of(own: Option) -> ([f64; METRIC_COUNT], f64) { + let mut error_bound = [0.0; METRIC_COUNT]; + let Some(own) = own else { + return (error_bound, 0.0); + }; + let m = AccuracyMetric::ALL + .iter() + .position(|&listed| listed == own.metric) + .expect("AccuracyMetric::ALL lists every metric"); + error_bound[m] = own.max_error; + (error_bound, own.failure_probability) +} + +/// What (K) is built over, per execution (by position in +/// `SelectableExecutions::executions`), before any variable: its +/// selectable input edges and upper bounds on the error and failure +/// probability any selection gives it (the big-M of (K)). +/// +/// Only an execution that is its own original (not in a copy of an +/// equivalent context, [`SharedVariables`]) and that a query with an +/// accuracy target reaches has rows: a copy's rows would repeat its +/// original's on the same variables. An execution whose inputs carry no +/// error under any selection (a readout, whose state is exact) has a fixed +/// error, its own, and no variable: its consumers read it as a constant +/// times $y$ ([`Composed`]). +/// +/// Example: B = p99 / p50 at relative 0.03, both read from DDSketch +/// readouts required at relative 0.02 or 0.01: each readout's own error is +/// its requirement, the ratio adds none and passes relative error on with +/// factor 1/(1 − 0.1) per input, so its ceiling is 1.111 (0.02 + 0.02). +pub(super) struct ErrorModel<'a> { + /// The rules (K) applies; `None` when no query has a target. + rules: Option<&'a AccuracyRules>, + /// Whether (K) has rows for the execution. + has_rows: Vec, + /// Per input, the selectable edges into the execution that have a + /// $y$: (position in `SelectableExecutions::edges`, producer + /// execution); empty for an execution without rows. + inputs: Vec>>, + /// $M$ per metric: the largest $E_{c,m}$ any selection gives. + error_bound_ceiling: Vec<[f64; METRIC_COUNT]>, + /// The largest $D_c$ any selection gives. + failure_probability_ceiling: Vec, + /// Whether the execution's error is fixed: its inputs carry none under + /// any selection, so its ceilings are its own error. + fixed: Vec, +} + +/// An execution's error in one metric, or its failure probability, as +/// (K) reads it: fixed (a readout's own requirement) or a variable with +/// its ceiling. +#[derive(Clone, Copy)] +enum Composed { + Fixed(f64), + Variable(Variable, f64), +} + +impl<'a> ErrorModel<'a> { + /// The model over `inputs` and `rules`, with `y` the edge variables + /// kept and `shared` the copies of equivalent contexts. + pub(super) fn new( + inputs: &SelectionInputs<'_>, + rules: Option<&'a AccuracyRules>, + y: &BTreeMap, + shared: &SharedVariables, + ) -> Self { + let selectable = inputs.selectable; + let count = selectable.executions().len(); + let has_rows: Vec = (0..count) + .map(|index| { + rules.is_some_and(|rules| rules.targeted[index]) + && shared.original_of_execution[index] == index + }) + .collect(); + let inputs_of = |index: usize| -> Vec> { + if !has_rows[index] { + return Vec::new(); + } + (0..selectable.input_count(index)) + .map(|input| { + selectable + .edges_into(index, input) + .iter() + .filter(|position| y.contains_key(position)) + .map(|&position| (position, selectable.edges()[position].producer)) + .collect() + }) + .collect() + }; + let mut model = Self { + rules, + inputs: (0..count).map(inputs_of).collect(), + has_rows, + error_bound_ceiling: vec![[0.0; METRIC_COUNT]; count], + failure_probability_ceiling: vec![0.0; count], + fixed: vec![false; count], + }; + model.set_ceilings(inputs); + model.fixed = (0..count) + .map(|c| { + model.has_rows[c] + && model.inputs[c].iter().flatten().all(|&(_, u)| { + model.failure_probability_ceiling[u] == 0.0 + && model.error_bound_ceiling[u].iter().all(|&e| e == 0.0) + }) + }) + .collect(); + model + } + + /// Execution `u`'s error in metric `m` (by position in + /// `AccuracyMetric::ALL`); `None` when it is 0 under every selection. + fn error_bound(&self, accuracy: &AccuracyVariables, u: usize, m: usize) -> Option { + let ceiling = self.error_bound_ceiling[u][m]; + match (ceiling > 0.0, self.fixed[u]) { + (false, _) => None, + (true, true) => Some(Composed::Fixed(ceiling)), + (true, false) => Some(Composed::Variable(accuracy.error_bound[&(u, m)], ceiling)), + } + } + + /// Execution `u`'s failure probability; `None` when it is 0 under + /// every selection. + fn failure_probability(&self, accuracy: &AccuracyVariables, u: usize) -> Option { + let ceiling = self.failure_probability_ceiling[u]; + match (ceiling > 0.0, self.fixed[u]) { + (false, _) => None, + (true, true) => Some(Composed::Fixed(ceiling)), + (true, false) => Some(Composed::Variable( + accuracy.failure_probability[&u], + ceiling, + )), + } + } + + /// The ceilings, by rounds of the composition rule of (K) over the + /// selectable edges until none grows: at most one round per execution, + /// the longest path a selection without cycles (constraint (E)) has. + fn set_ceilings(&mut self, inputs: &SelectionInputs<'_>) { + let Some(rules) = self.rules else { + return; + }; + let with_rows: Vec = (0..self.has_rows.len()) + .filter(|&c| self.has_rows[c]) + .collect(); + let own: HashMap = with_rows + .iter() + .map(|&c| (c, own_error_of(own_error(inputs, c)))) + .collect(); + for _ in 0..=with_rows.len() { + let mut grew = false; + for &c in &with_rows { + let (mut error_bound, mut failure_probability) = own[&c]; + for (i, edges) in self.inputs[c].iter().enumerate() { + for (m, bound) in error_bound.iter_mut().enumerate() { + if let Some(contract) = rules.contract(c, m) { + let largest = edges + .iter() + .map(|&(_, u)| self.error_bound_ceiling[u][m]) + .fold(0.0, f64::max); + *bound += contract.error_factor_per_input[i] * largest; + } + } + failure_probability += edges + .iter() + .map(|&(_, u)| self.failure_probability_ceiling[u]) + .fold(0.0, f64::max); + } + for (ceiling, bound) in self.error_bound_ceiling[c].iter_mut().zip(error_bound) { + if bound > *ceiling { + *ceiling = bound; + grew = true; + } + } + if failure_probability > self.failure_probability_ceiling[c] { + self.failure_probability_ceiling[c] = failure_probability; + grew = true; + } + } + if !grew { + break; + } + } + } +} + +/// The accuracy variables of constraint (K), all continuous, each only for +/// an execution with rows ([`ErrorModel`]) whose error is not fixed and +/// whose ceiling is above 0, by execution position. +pub(super) struct AccuracyVariables { + /// $E_{\kappa,c,m}$: the execution's error bound in metric $m$ (by + /// position in `AccuracyMetric::ALL`). + error_bound: HashMap<(usize, usize), Variable>, + /// $D_{\kappa,c}$: its failure probability. + failure_probability: HashMap, +} + +impl AccuracyVariables { + /// $E$ and $D$ over `model`, added to `problem`. + pub(super) fn new(model: &ErrorModel<'_>, problem: &mut ProblemVariables) -> Self { + let mut variables = Self { + error_bound: HashMap::new(), + failure_probability: HashMap::new(), + }; + for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.fixed[c]) { + for m in 0..METRIC_COUNT { + if model.error_bound_ceiling[c][m] > 0.0 { + variables + .error_bound + .insert((c, m), problem.add(variable().min(0))); + } + } + if model.failure_probability_ceiling[c] > 0.0 { + variables + .failure_probability + .insert(c, problem.add(variable().min(0))); + } + } + variables + } + + /// The number of $E$ and $D$ variables. + #[cfg(test)] + pub(super) fn variable_count(&self) -> usize { + self.error_bound.len() + self.failure_probability.len() + } + + /// Whether (K) has no variable: no query with an accuracy target reads + /// an error. + pub(super) fn is_empty(&self) -> bool { + self.error_bound.is_empty() && self.failure_probability.is_empty() + } +} + +/// (K) (section 4, "Query composition and root checks"), per execution +/// $c$ with rows ([`ErrorModel`]): +/// - $E_{c,m} \ge \varepsilon_c[m = m_c] + \sum_i \beta_{c,i,m} +/// F_{c,i,m}$ and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per +/// selectable edge, $F_{c,i,m} \le b_{c,i}$ where the contract's linear +/// bound holds only up to $b_{c,i}$ (a ratio's denominator), and +/// $E_{u,m} \le M_{u,m}(1 - y_{u,c,i})$ when $c$ has no contract in $m$ +/// (the metric rule on the selected path); an input whose error does not +/// reach the output ($\beta = 0$, a count's) passes none; +/// - $D_c \ge \delta_c + \sum_i H_{c,i}$, $H_{c,i} \ge D_u - M_u(1 - +/// y_{u,c,i})$ (a union bound); +/// - per query $q$ with a target, output group $g$ its root reads and +/// execution $v$ answering it, $E_{v,m} \le \varepsilon_q[m \in +/// \text{allowed}] + M(1 - z_{q,v})$, the metrics $g$ allows at the root +/// ([`query_roots_with_target`]: the target's at an operator root, none +/// for the operands of a scalar expression no contract covers), and +/// $D_v \le \delta_q + M(1 - z_{q,v})$ with $\delta_q$ per value +/// (`AccuracyTarget::failure_probability_per_value`); no $D$ row without +/// a per-value $\delta_q$. Each limit is widened by [`tolerated_limit`], +/// as the edge check widens it. +/// +/// $F$ and $H$ are added to `problem` here. A copy of an equivalent context +/// and its queries have no rows: they share their original's variables, +/// and the copy's rows would be its original's +/// ([`super::equivalent_contexts::copies_in_equivalent_contexts`] pairs +/// queries only with equal targets, $n_q$ and metrics allowed per output). +pub(super) fn accuracy_constraints( + inputs: &SelectionInputs<'_>, + model: &ErrorModel<'_>, accuracy: &AccuracyVariables, + y: &BTreeMap, + z: &BTreeMap<(QueryId, usize), Variable>, + shared: &SharedVariables, + problem: &mut ProblemVariables, ) -> Vec { - let _ = (physical, selection, accuracy); - todo!("#1207 PR 8: constraint (K)") + let Some(rules) = model.rules else { + return Vec::new(); + }; + let mut constraints = Vec::new(); + for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.fixed[c]) { + // Σ_i β_{c,i,m} F_{c,i,m} per metric, and Σ_i H_{c,i}. + let mut error_of_inputs: [Expression; METRIC_COUNT] = Default::default(); + let mut failure_probability_of_inputs = Expression::default(); + for (i, edges) in model.inputs[c].iter().enumerate() { + for (m, error_of_input) in error_of_inputs.iter_mut().enumerate() { + let read: Vec<(Variable, Composed)> = edges + .iter() + .filter_map(|&(position, u)| { + Some((y[&position], model.error_bound(accuracy, u, m)?)) + }) + .collect(); + if read.is_empty() { + continue; + } + let Some(contract) = rules.contract(c, m) else { + // The metric rule: no error in m reaches c. + for (y, error) in read { + constraints.push(match error { + Composed::Fixed(_) => constraint!(y <= 0), + Composed::Variable(e, ceiling) => constraint!(e <= ceiling * (1 - y)), + }); + } + continue; + }; + if !contract.input_error_reaches_output(i) { + continue; + } + let of_input = chosen_producers(&read, &mut constraints, problem); + if let Some(holds_up_to) = contract.linear_bound_holds_up_to_per_input[i] { + let holds_up_to = tolerated_limit(holds_up_to); + constraints.push(constraint!(of_input.clone() <= holds_up_to)); + } + *error_of_input += contract.error_factor_per_input[i] * of_input; + } + let read: Vec<(Variable, Composed)> = edges + .iter() + .filter_map(|&(position, u)| { + Some((y[&position], model.failure_probability(accuracy, u)?)) + }) + .collect(); + if !read.is_empty() { + failure_probability_of_inputs += chosen_producers(&read, &mut constraints, problem); + } + } + let (own_error_bound, own_failure_probability) = own_error_of(own_error(inputs, c)); + for (m, of_inputs) in error_of_inputs.into_iter().enumerate() { + if let Some(&error_bound) = accuracy.error_bound.get(&(c, m)) { + let own = own_error_bound[m]; + constraints.push(constraint!(error_bound >= own + of_inputs)); + } + } + if let Some(&failure_probability) = accuracy.failure_probability.get(&c) { + constraints.push(constraint!( + failure_probability >= own_failure_probability + failure_probability_of_inputs + )); + } + } + let memo = &inputs.physical.logical_candidates.memo; + for (&(query, group), (query_id, executions)) in memo + .output_groups() + .iter() + .zip(inputs.selectable.answering()) + { + debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); + if shared.original_of_query[query] != *query_id { + continue; + } + let Some(limits) = rules.roots.get(&(query, group)) else { + continue; + }; + for &v in executions { + let z = z[&(*query_id, v)]; + for (m, metric) in AccuracyMetric::ALL.iter().enumerate() { + let limit = match limits.allowed_metrics.contains(metric) { + true => tolerated_limit(limits.max_error), + false => 0.0, + }; + constraints.extend( + model + .error_bound(accuracy, v, m) + .and_then(|error| at_most_when_answering(error, limit, z)), + ); + } + if let (Some(limit), Some(failure_probability)) = ( + limits.failure_probability_per_value, + model.failure_probability(accuracy, v), + ) { + constraints.extend(at_most_when_answering( + failure_probability, + tolerated_limit(limit), + z, + )); + } + } + } + constraints +} + +/// The error (or failure probability) one input of a consumer reads from +/// the producer selected on it, `read` listing each producer's with its +/// $y$: $\sum_u e_u y_u$ over the producers with a fixed error (at most one +/// $y$ of an input is 1, constraint (A)), and over the others a variable +/// $F \ge E_u - M_u(1 - y_u)$, also at least that sum. +fn chosen_producers( + read: &[(Variable, Composed)], + constraints: &mut Vec, + problem: &mut ProblemVariables, +) -> Expression { + let fixed: Expression = read + .iter() + .filter_map(|&(y, composed)| match composed { + Composed::Fixed(value) => Some(value * y), + Composed::Variable(..) => None, + }) + .sum(); + if read + .iter() + .all(|(_, composed)| matches!(composed, Composed::Fixed(_))) + { + return fixed; + } + let of_input = problem.add(variable().min(0)); + constraints.push(constraint!(of_input >= fixed)); + for &(y, composed) in read { + if let Composed::Variable(value, ceiling) = composed { + constraints.push(constraint!(of_input >= value - ceiling * (1 - y))); + } + } + of_input.into() +} + +/// The root row bounding `composed` by `limit` when $z = 1$: none when the +/// ceiling is within it; $z = 0$ for a fixed value above it. +fn at_most_when_answering(composed: Composed, limit: f64, z: Variable) -> Option { + match composed { + Composed::Fixed(value) => (value > limit).then(|| constraint!(z <= 0)), + Composed::Variable(value, ceiling) => { + (limit < ceiling).then(|| constraint!(value <= limit + (ceiling - limit) * (1 - z))) + } + } } diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs index f48352bb0..784b6cfba 100644 --- a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -3,8 +3,9 @@ //! ([`copies_in_equivalent_contexts`]), so HiGHS does not branch over each //! copy. -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use asap_accuracy::query_roots_with_target; use asap_types::accuracy::AccuracyTarget; use asap_types::stages::{ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId}; @@ -73,6 +74,14 @@ struct QuerySignature<'a> { /// Its accuracy target, by index among the workload's distinct targets /// ([`AccuracyTarget`] holds floats: metric, ε, δ and scope all equal). accuracy_target_index: usize, + /// $n_q$ (`MemoQuery::number_of_values_per_query_evaluation`), which + /// splits its target's failure probability per value in (K). + number_of_values_per_query_evaluation: Option, + /// Per output, whether the group it reads allows the target's metrics + /// at the root (`asap_accuracy::query_roots_with_target`: an operator + /// root does, an operand of `scalar(...) * 2` does not), which (K)'s + /// root rows read. + allows_target_metrics_per_output: Vec, } /// Which executions, edges and queries share variables @@ -82,8 +91,9 @@ struct QuerySignature<'a> { /// they have the same [`ContextKey`] (the same candidates, kept alike, the /// same memo edges from producers that correspond, in the context or the /// same execution outside every evaluation context, and queries that pair -/// up with the same output executions, exact-only groups, span and -/// accuracy target, so the accuracy constraint holds alike in both) and +/// up with the same output executions, exact-only groups, span, accuracy +/// target, $n_q$ and metrics allowed at each output, so the accuracy +/// constraint (K) has the same rows in both) and /// every cost the MILP reads in one (of a kept execution, or of an edge /// between two kept ones) is the same multiple $\alpha > 0$ of the /// other's, to [`COST_RELATIVE_TOLERANCE`] of the copy's largest such @@ -131,9 +141,25 @@ pub(crate) fn copies_in_equivalent_contexts( own.push(index); } } - let mut outputs_of: BTreeMap> = BTreeMap::new(); - for (query, answering) in selectable.answering() { - outputs_of.entry(*query).or_default().push(answering); + let memo = &inputs.physical.logical_candidates.memo; + // The (query, output group) pairs that allow the target's metrics. + let allowing_target_metrics: HashSet<(usize, MemoGroupId)> = query_roots_with_target(memo) + .into_iter() + .filter(|(_, _, _, metrics)| !metrics.is_empty()) + .map(|(query, group, _, _)| (query, group)) + .collect(); + // Per query, each output's answering executions and whether it allows + // the target's metrics. + let mut outputs_of: BTreeMap> = BTreeMap::new(); + for (&(query, group), (query_id, answering)) in + memo.output_groups().iter().zip(selectable.answering()) + { + debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); + let allows = allowing_target_metrics.contains(&(query, group)); + outputs_of + .entry(*query_id) + .or_default() + .push((answering, allows)); } // Each evaluation context's queries; a query without an output has // none. @@ -176,7 +202,7 @@ pub(crate) fn copies_in_equivalent_contexts( // A query has an evaluation context exactly when it has an output reference. let outputs = outputs_of[&query] .iter() - .map(|answering| { + .map(|(answering, _)| { answering .iter() .map(|&index| in_context(context, index)) @@ -185,6 +211,12 @@ pub(crate) fn copies_in_equivalent_contexts( .collect::>>()?; Some(QuerySignature { answering_executions_per_output: outputs, + number_of_values_per_query_evaluation: memo.queries()[query.0] + .number_of_values_per_query_evaluation, + allows_target_metrics_per_output: outputs_of[&query] + .iter() + .map(|&(_, allows)| allows) + .collect(), exact_only_groups: inputs .physical .logical_candidates diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 1394a0982..e91cec076 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -7,11 +7,9 @@ //! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_rows`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | //! | retention (section 5) | $h_v$ | (R) [`super::retention::retention_constraints`] | //! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | -//! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | +//! | accuracy (section 4), per execution | $E_{\kappa,c,m}$, $D_{\kappa,c}$ ([`AccuracyVariables`]); $F_{\kappa,c,i,m}$, $H_{\kappa,c,i}$ | (K) [`accuracy_constraints`], over [`ErrorModel`] | //! -//! [`solve_selection`] solves it without (K), whose variables and -//! constraint are in [`super::accuracy`], `todo!()` until it is added. The -//! strongly connected components (E) needs are in [`super::graph`]; the +//! The strongly connected components (E) needs are in [`super::graph`]; the //! executions left out before scaling, those dearer than a feasible plan //! and those their `OneEvaluation` twin dominates, in //! [`super::upper_bound`] and [`super::dominance`]. @@ -28,6 +26,7 @@ use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; +use super::accuracy::{accuracy_constraints, AccuracyRules, AccuracyVariables, ErrorModel}; use super::dominance::dominated_by_one_evaluation_twin; use super::equivalent_contexts::{copies_in_equivalent_contexts, SharedVariables}; use super::exact_only::{exact_only_constraints, exact_only_variables}; @@ -60,9 +59,9 @@ pub(crate) struct SelectionVariables { x: Vec, /// $y_{\kappa,u,c,i}$, by edge, on the edges costed between executions /// kept, in edge order. - y: BTreeMap, + pub(super) y: BTreeMap, /// $z_{q,v}$, by query and the execution answering it, in key order. - z: BTreeMap<(QueryId, usize), Variable>, + pub(super) z: BTreeMap<(QueryId, usize), Variable>, /// $t$, by execution: only for executions on a cycle of selectable /// edges (a strongly connected component); `None` elsewhere. t: Vec>, @@ -82,11 +81,12 @@ pub(crate) struct SelectionVariables { /// fixed at 0 for an execution that is not `kept` (not costed, or dearer /// than a feasible plan), and no $y$ on an edge of one. The executions, /// edges and queries of a copy of an equivalent evaluation context take the -/// original's $x$, $y$ and $z$ ([`copies_in_equivalent_contexts`]), so -/// their costs add up on one variable. +/// original's $x$, $y$ and $z$ (`shared`, [`copies_in_equivalent_contexts`]), +/// so their costs add up on one variable. fn selection_variables( inputs: &SelectionInputs<'_>, kept: &[bool], + shared: &SharedVariables, components: &Components, problem: &mut ProblemVariables, ) -> SelectionVariables { @@ -95,7 +95,7 @@ fn selection_variables( original_of_execution, original_of_edge, original_of_query, - } = copies_in_equivalent_contexts(inputs, kept); + } = shared; let mut x: Vec = Vec::with_capacity(kept.len()); for (index, &kept) in kept.iter().enumerate() { let variable = match (original_of_execution[index], kept) { @@ -390,7 +390,7 @@ pub(crate) enum SelectionMilpError { /// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} /// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a /// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], -/// [`raw_data_disk_rows`], (R) and (X), with HiGHS at gap 0 on one +/// [`raw_data_disk_rows`], (R), (X) and (K), with HiGHS at gap 0 on one /// thread. The executions dearer than a feasible plan /// ([`dearer_than_a_feasible_plan`]) or dominated by their `OneEvaluation` /// twin ([`dominated_by_one_evaluation_twin`]) are left out first (and a @@ -404,15 +404,28 @@ pub(crate) fn solve_selection( ) -> Result { let started = Instant::now(); let costs = inputs.costs; - let kept: Vec = dearer_than_a_feasible_plan(inputs) + let rules = AccuracyRules::of(inputs); + let kept: Vec = dearer_than_a_feasible_plan(inputs, rules.as_ref()) .iter() .zip(dominated_by_one_evaluation_twin(inputs)) .zip(&costs.per_execution) .map(|((&dearer, dominated), cost)| cost.is_ok() && !dearer && !dominated) .collect(); let components = Components::of(inputs); + let shared = copies_in_equivalent_contexts(inputs, &kept); let mut problem = ProblemVariables::new(); - let variables = selection_variables(inputs, &kept, &components, &mut problem); + let variables = selection_variables(inputs, &kept, &shared, &components, &mut problem); + let error_model = ErrorModel::new(inputs, rules.as_ref(), &variables.y, &shared); + let accuracy = AccuracyVariables::new(&error_model, &mut problem); + let accuracy_rows = accuracy_constraints( + inputs, + &error_model, + &accuracy, + &variables.y, + &variables.z, + &shared, + &mut problem, + ); // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs @@ -467,6 +480,7 @@ pub(crate) fn solve_selection( &variables.z, ), exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), + accuracy_rows, ] .into_iter() .flatten() @@ -500,6 +514,13 @@ pub(crate) fn solve_selection( model.set_option("mip_abs_gap", 0.0); model.set_option("output_flag", false); model.set_option("threads", 1); + if !accuracy.is_empty() { + // Errors and failure probabilities are small (1e-3, 1e-7): the + // default feasibility tolerance 1e-7 would let (K) pass a bound + // above its limit. + model.set_option("primal_feasibility_tolerance", 1e-10); + model.set_option("mip_feasibility_tolerance", 1e-10); + } let construction_ms = started.elapsed().as_millis() as u64; let started = Instant::now(); let solved = model diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index f5b68ae7b..517877ee3 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -7,20 +7,28 @@ use std::collections::{BTreeSet, HashMap}; use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::scalar::ExprSemantics; +use asap_types::ir::scalar::ScalarValue; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; +use asap_types::ir::{QueryRoot, ScalarExpr}; use asap_types::stages::{ - CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, + CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalRequirements, + Memo, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, RequiredAccuracy, }; use asap_types::stages::{Lifetime, PhysicalProperties, StorageMedium}; -use asap_types::summary_type::{ExactKind, ExactParams, SketchAlgorithm, SummaryUpdate}; +use asap_types::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, SummaryUpdate, +}; use asap_types::test_support::memo_builders::{ - group, own_candidate, quantile_over_time_readouts, query, scan_group, try_candidate, + group, own_candidate, quantile_over_time_readouts, query, scan_group, sketch_state, + try_candidate, }; use asap_types::test_support::memo_fixtures; +use asap_types::test_support::memo_fixtures::arithmetic; use asap_types::workload::DataWorkload; use asap_types::workload::{ Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, @@ -29,6 +37,7 @@ use asap_types::workload::{ use super::*; use crate::cost::ExecutionCosts; +use crate::milp::accuracy::{AccuracyRules, AccuracyVariables, ErrorModel}; use crate::plan::plan_of; use crate::retention::{edge_retention_need_ms, least_retention_ms_per_execution}; use crate::selectable::{QueryId, SelectableEdge}; @@ -859,7 +868,7 @@ fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { selectable: &selectable, costs: &costs, }; - let left_out = dearer_than_a_feasible_plan(&inputs); + let left_out = dearer_than_a_feasible_plan(&inputs, None); let unused: Vec = selectable .executions() .iter() @@ -893,12 +902,15 @@ fn the_bound_counts_the_least_retention_an_execution_pays() { let entries = every_10s(1); let (selectable, costs) = selection_inputs(&physical, &entries, &[1.0, 2.0, 1.0], &[], &[(1, 1e6)]); - let left_out = dearer_than_a_feasible_plan(&SelectionInputs { - physical: &physical, - entries: &entries, - selectable: &selectable, - costs: &costs, - }); + let left_out = dearer_than_a_feasible_plan( + &SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }, + None, + ); let left_out: Vec = selectable .executions() .iter() @@ -1198,3 +1210,334 @@ fn a_left_out_execution_does_not_set_the_tolerance_of_context_copies() { ); assert_eq!(candidates(&solved), vec![0, 1, 2, 2]); } + +// Constraint (K). + +/// Quantile `q` over a DDSketch state of `m[1m]` (one build per listed α): +/// the readout group, one readout per relative-error requirement of +/// `levels` (ε, δ). +fn ddsketch_readouts( + memo: &mut Memo, + q: f64, + levels: &[(f64, f64)], +) -> (MemoGroupId, Vec) { + let m = scan(memo); + let window = range(memo, 60, m); + let state = sketch_state( + memo, + window, + SketchAlgorithm::DDSketch, + Reduction::PerEntity, + ); + let operator = memo_fixtures::readout(SketchStatistic::Quantile { q }, state); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let readouts = levels + .iter() + .map(|&(max_error, failure_probability)| { + let mut readout = CandidateForMemoGroup::new( + operator.clone(), + None, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ); + readout.input_requirements = vec![LogicalRequirements { + required_accuracy: Some(RequiredAccuracy { + metric: AccuracyMetric::Relative, + max_error, + failure_probability, + }), + }]; + memo.add_candidate(group, readout).unwrap() + }) + .collect(); + (group, readouts) +} + +/// Every candidate of `memo` at query time, `OneEvaluation`. +fn every_candidate(memo: &Memo) -> Vec<(CandidateForMemoGroupId, Lifetime)> { + memo.groups() + .iter() + .enumerate() + .flat_map(|(group, memo_group)| { + (0..memo_group.candidates.len()).map(move |index_in_group| { + ( + CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }, + ONE_EVALUATION, + ) + }) + }) + .collect() +} + +/// The physical candidate of logical candidate `id` in `physical`. +fn physical_of(physical: &PhysicalASAPMemoCandidates, id: CandidateForMemoGroupId) -> usize { + physical + .physical_candidates + .iter() + .position(|candidate| candidate.logical_candidate_id == id) + .unwrap() +} + +fn relative(max_error: f64) -> AccuracyTarget { + AccuracyTarget::max_error(max_error).accuracy_metric(AccuracyMetric::Relative) +} + +/// #1207's query B, p99 / p50 at relative 0.03 (concepts doc, section 4, +/// "Example: allocation in a ratio"), over readouts required at relative +/// 0.02 (cost 1) or 0.01 (cost 3) each: 1.111 (0.02 + 0.02) = 0.044 and +/// 1.111 (0.02 + 0.01) = 0.033 are above 0.03, so (K) selects both 0.01 +/// readouts (0.022). The greedy plan reads the two 0.02 readouts (cost 2) +/// and misses the target, so it bounds nothing: as a bound it would leave +/// out each 0.01 readout (cost 3 > 2). The same ratio every 10 s and every +/// 20 s is one context's copy, whose (K) has no rows of its own and +/// selects alike. +#[test] +fn k_allocates_a_ratio_s_error_among_its_readouts() { + for entries in [vec![every(10_000)], vec![every(10_000), every(20_000)]] { + let mut memo = empty_memo(); + let levels = [(0.02, 1.0), (0.01, 1.0)]; + let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); + let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + for _ in &entries { + query(&mut memo, ratio, Some(relative(0.03))); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let mut w = vec![0.0; physical.physical_candidates.len()]; + for readouts in [&p99_readouts, &p50_readouts] { + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + } + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + // Each cost in the 20 s context is half the 10 s one's. + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if let asap_types::stages::ExecutionContext::Evaluation { grid } = + selectable.context(selectable.executions()[index].context) + { + if grid.period_ms == 20_000 { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + } + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let solved = solve_selection(&inputs).unwrap(); + let selected = candidates(&solved); + for readouts in [&p99_readouts, &p50_readouts] { + let chosen = physical_of(&physical, readouts[1]); + let not_chosen = physical_of(&physical, readouts[0]); + assert_eq!( + selected.iter().filter(|&&v| v == chosen).count(), + entries.len(), + "{selected:?}" + ); + assert!(!selected.contains(¬_chosen), "{selected:?}"); + } + // (K)'s E and D exist only in the 10 s context (the 20 s one is + // its copy), and only for the ratio: a readout's error is fixed. + let kept = vec![true; selectable.executions().len()]; + let shared = copies_in_equivalent_contexts(&inputs, &kept); + let mut problem = ProblemVariables::new(); + let variables = selection_variables( + &inputs, + &kept, + &shared, + &Components::of(&inputs), + &mut problem, + ); + let rules = AccuracyRules::of(&inputs); + let model = ErrorModel::new(&inputs, rules.as_ref(), &variables.y, &shared); + let accuracy = AccuracyVariables::new(&model, &mut problem); + let one_context = 2; + assert_eq!(accuracy.variable_count(), one_context, "{entries:?}"); + } +} + +/// A query reading one readout group of quantile 0.5 whose readouts have +/// `levels`, rooted at `root` over it, with `target` and $n_q$ +/// `values_per_evaluation`: whether some plan meets (K). +fn plans( + target: Option, + levels: &[(f64, f64)], + values_per_evaluation: Option, + root: fn(MemoGroupId) -> QueryRoot, +) -> bool { + let mut memo = empty_memo(); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, levels); + memo.add_query(MemoQuery { + root: root(readout_group), + accuracy_target: target, + number_of_values_per_query_evaluation: values_per_evaluation, + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + match solve(&physical, &every_10s(1), &w, &[], &[]).0 { + Ok(_) => true, + Err(SelectionMilpError::Infeasible) => false, + Err(other) => panic!("{other:?}"), + } +} + +/// The root accepts error only in the target's metric: a readout off by +/// relative 0.01 meets relative 0.01, not relative 0.005 nor rank 0.01; a +/// target naming no metric accepts any. +#[test] +fn k_accepts_error_only_in_the_target_s_metric() { + let level = [(0.01, 1.0)]; + let operator = QueryRoot::Operator; + assert!(plans(Some(relative(0.01)), &level, Some(1), operator)); + assert!(!plans(Some(relative(0.005)), &level, Some(1), operator)); + let rank = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank); + assert!(!plans(Some(rank), &level, Some(1), operator)); + let any = AccuracyTarget::max_error(0.01); + assert!(plans(Some(any), &level, Some(1), operator)); +} + +/// A whole-result failure probability 0.01 over 10 values is 1e-3 per +/// value, which a readout failing with probability 1e-3 meets; over 100 +/// values (1e-4 per value) it does not. With the count unknown no +/// per-value δ follows, and (K) has no row for it (Pass 2 gives such a +/// query's readouts no level). +#[test] +fn k_divides_a_whole_result_failure_probability_over_the_values() { + let level = [(0.01, 1e-3)]; + let target = relative(0.01).failure_probability(0.01); + let operator = QueryRoot::Operator; + assert!(plans(Some(target.clone()), &level, Some(10), operator)); + assert!(!plans(Some(target.clone()), &level, Some(100), operator)); + assert!(plans(Some(target), &level, None, operator)); +} + +/// `scalar(v)` returns the value it reads, so its root accepts the +/// target's metric; `scalar(v) * 2`'s operand is read exactly, as no error +/// contract covers the scalar expression; without a target either reads +/// the readout. +#[test] +fn a_scalar_root_accepts_the_target_s_metric_only_where_it_returns_its_operand() { + let level = [(0.01, 1.0)]; + let scalar = |group| QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(group)); + let doubled = |group| { + QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::PromqlScalarFromVector(group)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }) + }; + assert!(plans(Some(relative(0.01)), &level, Some(1), scalar)); + assert!(!plans(Some(relative(0.01)), &level, Some(1), doubled)); + assert!(plans(None, &level, Some(1), doubled)); +} + +/// A ratio's linear bound holds while its denominator is off by at most +/// 0.1 (`ErrorContract::linear_bound_holds_up_to_per_input`): a +/// denominator readout at relative 0.2 cannot be read under any target, +/// one at 0.05 can. +#[test] +fn k_caps_a_ratio_s_denominator_at_its_validity_cap() { + let ratio_plans = |denominator_error: f64| { + let mut memo = empty_memo(); + let (p99, _) = ddsketch_readouts(&mut memo, 0.99, &[(0.01, 1.0)]); + let (p50, _) = ddsketch_readouts(&mut memo, 0.5, &[(denominator_error, 1.0)]); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + query(&mut memo, ratio, Some(relative(1.0))); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + match solve(&physical, &every_10s(1), &w, &[], &[]).0 { + Ok(_) => true, + Err(SelectionMilpError::Infeasible) => false, + Err(other) => panic!("{other:?}"), + } + }; + assert!(ratio_plans(0.05)); + assert!(!ratio_plans(0.2)); +} + +/// The same readout every 10 s and every 20 s at relative 0.01 with δ = +/// 0.01 over the whole result, returning 10 values every 10 s and 100 +/// every 20 s: the readout failing with 1e-3 (cost 1) meets 1e-3 per +/// value, not 1e-4, where the one failing with 1e-5 (cost 2) is needed. +/// The contexts differ in what (K) reads ($n_q$), so the 20 s one is not a +/// copy: each selects its own readout. +#[test] +fn contexts_whose_queries_split_their_failure_probability_differently_are_not_copies() { + let mut memo = empty_memo(); + let (readout_group, readouts) = + ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1e-3), (0.01, 1e-5)]); + for values in [10, 100] { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(values), + }); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + // The scan costs 10, so neither readout is dearer than a feasible plan. + let mut w = vec![0.0; physical.physical_candidates.len()]; + w[0] = 10.0; + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 2.0; + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if let asap_types::stages::ExecutionContext::Evaluation { grid } = + selectable.context(selectable.executions()[index].context) + { + if grid.period_ms == 20_000 { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + } + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + let selected = candidates(&solved); + assert!( + selected.contains(&physical_of(&physical, readouts[0])), + "{selected:?}" + ); + assert!( + selected.contains(&physical_of(&physical, readouts[1])), + "{selected:?}" + ); +} + +/// The same readout group as `q` every 10 s and as `scalar(q) * 2` every +/// 20 s, both at relative 0.01, with only a readout at relative 0.01: the +/// operator root accepts it, the scalar expression's operand only exact +/// values (no error contract covers it). The contexts differ in the +/// metrics their outputs allow, so the 20 s one is not a copy whose root +/// rows (K) could leave out: no plan meets both. +#[test] +fn contexts_whose_roots_allow_different_metrics_are_not_copies() { + let mut memo = empty_memo(); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1.0)]); + let doubled = QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::PromqlScalarFromVector(readout_group)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }); + for root in [QueryRoot::Operator(readout_group), doubled] { + memo.add_query(MemoQuery { + root, + accuracy_target: Some(relative(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let (solved, _) = solve(&physical, &[every(10_000), every(20_000)], &w, &[], &[]); + assert_eq!(solved.unwrap_err(), SelectionMilpError::Infeasible); +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 92da7901e..27c344621 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -14,6 +14,7 @@ use std::collections::{BTreeMap, HashMap}; +use super::accuracy::AccuracyRules; use super::model::{has_one_reader, selection_cost_per_second, SelectionInputs}; use super::COST_RELATIVE_TOLERANCE; use crate::selectable::QueryId; @@ -33,11 +34,15 @@ type TreeInputs = Vec>; /// can take off ($\min(0, c_{\kappa,u,v,i})$ per input) is over the /// bound. Such an execution is in no optimal plan, as long as every /// other selected execution costs at least 0 with its selected inputs. -/// None is ruled out when no feasible plan is found. -pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { +/// None is ruled out when no feasible plan is found, the accuracy +/// constraint (K) (`rules`) included. +pub(crate) fn dearer_than_a_feasible_plan( + inputs: &SelectionInputs<'_>, + rules: Option<&AccuracyRules>, +) -> Vec { let selectable = inputs.selectable; let count = selectable.executions().len(); - let Some(bound) = feasible_plan_cost_per_second(inputs) else { + let Some(bound) = feasible_plan_cost_per_second(inputs, rules) else { return vec![false; count]; }; let costs = inputs.costs; @@ -91,8 +96,15 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec) -> Option { +/// way, or when the plan found misses an accuracy target under (K) +/// (`rules`, [`AccuracyRules::selection_meets_targets`]): the cheapest +/// readouts of a ratio may each meet their level and compose above the +/// target, and a bound from such a plan could leave out the readouts the +/// optimum needs. +fn feasible_plan_cost_per_second( + inputs: &SelectionInputs<'_>, + rules: Option<&AccuracyRules>, +) -> Option { let selectable = inputs.selectable; let executions = selectable.executions(); let count = executions.len(); @@ -135,6 +147,11 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { continue 'repair; } } + if rules + .is_some_and(|rules| !rules.selection_meets_targets(inputs, &plan.edges, &answering)) + { + return None; + } let (cost, _) = selection_cost_per_second(inputs, &plan.executions, &plan.edges, &answering); return cost.is_finite().then_some(cost); diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index e999389ad..ae7c9f997 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -4,12 +4,15 @@ use std::collections::{BTreeSet, HashMap}; +use asap_accuracy::readout_error_metric; use asap_cost::node::analytical::AnalyticalCostError; use asap_cost::{summary_type_name, CostModel}; use asap_types::accuracy::AccuracyMetric; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; -use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::plan::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, +}; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidate, PrunedCandidate}; use thiserror::Error; @@ -112,8 +115,11 @@ impl RejectionCause { /// bound that met its target, and the sketches its output reads. #[derive(Debug, Clone, PartialEq)] pub struct AcceptedAccuracy { - /// The metric of the accepted error bound; `None` for an exact output, - /// which meets every metric, and for a query without an accuracy target. + /// Whether the query has an accuracy target. + pub has_accuracy_target: bool, + /// The metric of the accepted error bound, that of the first sketch + /// readout the output reads; `None` for an exact output, which meets + /// every metric, and for a query without an accuracy target. pub accepted_metric: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. @@ -121,9 +127,11 @@ pub struct AcceptedAccuracy { } impl std::fmt::Display for AcceptedAccuracy { - /// E.g. `accepted rank (Kll)`, or `accepted exact`. + /// E.g. `accepted rank (Kll)`, `accepted exact`, or `no accuracy + /// target (DDSketch)`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.accepted_metric { + None if !self.has_accuracy_target => write!(f, "no accuracy target")?, None => write!(f, "accepted exact")?, Some(metric) => { let name = serde_json::to_value(metric) @@ -224,12 +232,15 @@ pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { } /// Per query, the sketches its output reads in `plan` (each sketch build -/// below its root, in node order) and, when it reads one, its target's -/// metric. +/// below its root, in node order) and, when it has an accuracy target and +/// reads a sketch, the metric of the first sketch readout below its root +/// (`asap_accuracy::readout_error_metric`), whatever metric the target +/// names: a target naming none reports the one it accepted. pub(crate) fn accepted_accuracy( physical: &PhysicalASAPMemoCandidates, plan: &PhysicalASAPDAG, ) -> Vec { + let memo = &physical.logical_candidates.memo; let mut operator_roots = plan.roots.iter(); let mut producers_of: HashMap> = HashMap::new(); for edge in &plan.edges { @@ -238,10 +249,7 @@ pub(crate) fn accepted_accuracy( .or_default() .push(edge.producer); } - physical - .logical_candidates - .memo - .queries() + memo.queries() .iter() .enumerate() .map(|(query, memo_query)| { @@ -253,10 +261,45 @@ pub(crate) fn accepted_accuracy( Some((_, expression)) => expression.operator_refs().into_iter().copied().collect(), None => vec![*operator_roots.next().expect("one operator root per query")], }; - let sketches = sketches_below(plan, &producers_of, starts); + let mut sketches: Vec = Vec::new(); + let mut readout_metrics: Vec = Vec::new(); + for node in nodes_below(plan, &producers_of, starts) { + match &node.payload { + Operator::ASAP( + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, + ) if matches!(summary_type, FieldDataType::Sketch(_)) => { + let name = summary_type_name(summary_type); + if !sketches.contains(&name) { + sketches.push(name); + } + } + Operator::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { + // The algorithm of the state group the readout's + // candidate reads. + let algorithm = node.selected_physical_candidate.and_then(|v| { + let id = physical.physical_candidates[v.0].logical_candidate_id; + match memo.candidate(id).expression.operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, .. + }) => memo.groups()[summary_input.0 as usize] + .group_properties + .sketch_algorithm_of_state(), + _ => None, + } + }); + readout_metrics.extend(algorithm.and_then(|algorithm| { + readout_error_metric(&FieldDataType::Sketch(algorithm), query) + })); + } + _ => {} + } + } + let has_accuracy_target = memo_query.accuracy_target.is_some(); AcceptedAccuracy { - accepted_metric: (!sketches.is_empty()) - .then(|| memo_query.accuracy_target.as_ref()?.accuracy_metric) + has_accuracy_target, + accepted_metric: (has_accuracy_target && !sketches.is_empty()) + .then(|| readout_metrics.first().copied()) .flatten(), sketches, } @@ -264,13 +307,12 @@ pub(crate) fn accepted_accuracy( .collect() } -/// The summary type names of the sketch builds at or below `starts` in -/// `plan`, in node order, each once. -fn sketches_below( - plan: &PhysicalASAPDAG, +/// The nodes at or below `starts` in `plan`, in node order. +fn nodes_below<'a>( + plan: &'a PhysicalASAPDAG, producers_of: &HashMap>, starts: Vec, -) -> Vec { +) -> impl Iterator { let mut below = BTreeSet::new(); let mut pending = starts; while let Some(id) = pending.pop() { @@ -278,22 +320,9 @@ fn sketches_below( pending.extend(producers_of.get(&id).into_iter().flatten().copied()); } } - let mut sketches: Vec = Vec::new(); - for node in plan.nodes.iter().filter(|node| below.contains(&node.id)) { - if let Operator::ASAP( - ASAPOp::SummaryAgg { summary_type, .. } - | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, - ) = &node.payload - { - if matches!(summary_type, FieldDataType::Sketch(_)) { - let name = summary_type_name(summary_type); - if !sketches.contains(&name) { - sketches.push(name); - } - } - } - } - sketches + plan.nodes + .iter() + .filter(move |node| below.contains(&node.id)) } #[cfg(test)] diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs index 18d908020..48145973e 100644 --- a/crates/types/src/accuracy/target/mod.rs +++ b/crates/types/src/accuracy/target/mod.rs @@ -285,6 +285,26 @@ impl AccuracyTarget { self.max_error == 0.0 && self.failure_probability == Some(0.0) } + /// The target's failure probability per value: its δ split evenly over + /// the values its scope covers when one evaluation returns + /// `values_per_evaluation` ($n_q$; [`AccuracyScope::max_values_covered`]); + /// 1 when it sets no δ. `None` when that count is not known for a δ over + /// the whole result: no per-value δ follows (Pass 2 then gives the + /// query's readouts no level). + /// + /// Example: δ = 0.01 over the whole result of a query returning 100 + /// series gives 0.0001 per value; with the number of series unknown, + /// `None`; per value (`AccuracyScope::PerValue`), 0.01 either way. + pub fn failure_probability_per_value(&self, values_per_evaluation: Option) -> Option { + let Some(delta) = self.failure_probability else { + return Some(1.0); + }; + let values = self + .accuracy_scope + .max_values_covered(values_per_evaluation)?; + Some(delta / values.max(1) as f64) + } + /// The metrics this target accepts an error in: its metric, every /// metric when it names none. pub fn metrics(&self) -> Vec { diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs index d5f690d39..5c53a758a 100644 --- a/crates/types/src/accuracy/target/tests.rs +++ b/crates/types/src/accuracy/target/tests.rs @@ -332,3 +332,18 @@ fn exact_targets() { AccuracyScope::WholeResult ); } + +/// A whole-result δ splits over the values one evaluation returns; a +/// per-value δ does not; no δ is 1; an unknown count has none. +#[test] +fn a_whole_result_failure_probability_splits_over_the_values() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + assert_eq!(whole.failure_probability_per_value(Some(100)), Some(1e-4)); + assert_eq!(whole.failure_probability_per_value(None), None); + let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); + assert_eq!(per_value.failure_probability_per_value(None), Some(0.01)); + assert_eq!( + AccuracyTarget::max_error(0.01).failure_probability_per_value(None), + Some(1.0) + ); +} diff --git a/crates/types/src/stages/logical/group/mod.rs b/crates/types/src/stages/logical/group/mod.rs index cf087d8ea..9a6e39e03 100644 --- a/crates/types/src/stages/logical/group/mod.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -105,6 +105,21 @@ pub struct GroupProperties { } impl GroupProperties { + /// The `column_signs` entry of a state group's state column (its first + /// field that is not plain): the sign of the values its states were + /// updated with. `None` for a group without a state column. + /// + /// Example: a DDSketch state over `latency`, declared non-negative, + /// is `NonNegative`; over a gauge with no declared sign, `Unknown`. + pub fn state_column_sign(&self) -> Option { + let state = self + .schema + .fields + .iter() + .position(|field| !field.is_plain())?; + self.column_signs.get(state).copied() + } + /// The sketch algorithm of the state the group outputs; `None` for /// rows, values and exact states. pub fn sketch_algorithm_of_state(&self) -> Option { diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 7d52645b2..5b52d860c 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -125,6 +125,26 @@ impl Memo { } } + /// The most distinct items one distinct-count summary over `group`'s + /// rows counts in one evaluation: the sum of the bounds the data + /// workload declares for the sources scanned below it + /// (`GroupProperties::scanned_sources`, + /// `SourceFacts::max_distinct_per_evaluation`); `None` when one of them + /// declares none. + /// + /// Example: `count(distinct user_id)` over `Scan events` with `events` + /// bounded at 9,636 gives 9,636 for every group above the scan. + pub fn max_distinct_per_evaluation(&self, group: MemoGroupId) -> Option { + self.groups[group.0 as usize] + .group_properties + .scanned_sources + .iter() + .try_fold(0u64, |total, source| { + let (facts, _) = series_facts(&self.source_facts, source_name(source))?; + total.checked_add(facts.max_distinct_per_evaluation?) + }) + } + /// The groups the `by` keys of the state `group` is, or reads, form /// ([`Self::max_values_per_evaluation`]). fn declared_groups_of_state(&self, group: MemoGroupId) -> Option { diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 6d66f4ef1..0a40e109f 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -463,3 +463,39 @@ fn max_values_per_evaluation_bounds_a_readout_by_its_states_keys() { assert_eq!(memo.max_values_per_evaluation(quantile), Some(values)); } } + +/// A distinct count over `events`, bounded at 9,636 distinct items per +/// evaluation, counts at most 9,636 in every group above the scan; a join +/// of two bounded sources counts at most their sum; a source without a +/// bound bounds nothing. +#[test] +fn max_distinct_per_evaluation_sums_the_scanned_sources_bounds() { + let bounded = |bound: Option| crate::workload::SourceFacts { + max_distinct_per_evaluation: bound, + ..Default::default() + }; + let data = crate::workload::DataWorkload { + sources: [ + ("events".to_string(), bounded(Some(9_636))), + ("clicks".to_string(), bounded(Some(364))), + ("logs".to_string(), bounded(None)), + ] + .into(), + ..Default::default() + }; + let mut memo = Memo::new(&data); + let mut scan_of = |source: &str| memo.find_or_add_group(scan(source, &[], vec![])).unwrap(); + let (events, clicks, logs) = (scan_of("events"), scan_of("clicks"), scan_of("logs")); + let state = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Hll), events)) + .unwrap(); + assert_eq!(memo.max_distinct_per_evaluation(events), Some(9_636)); + assert_eq!(memo.max_distinct_per_evaluation(state), Some(9_636)); + assert_eq!(memo.max_distinct_per_evaluation(logs), None); + let both = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Concat { + children: vec![events, clicks], + })) + .unwrap(); + assert_eq!(memo.max_distinct_per_evaluation(both), Some(10_000)); +} diff --git a/crates/types/src/test_support/memo_builders.rs b/crates/types/src/test_support/memo_builders.rs index 09c5765d6..f9af4dc06 100644 --- a/crates/types/src/test_support/memo_builders.rs +++ b/crates/types/src/test_support/memo_builders.rs @@ -111,6 +111,23 @@ pub fn with_series(metric: &str, series_count: u64) -> DataWorkload { } } +/// A data workload declaring only that `metric`'s `value` is non-negative +/// (`SourceFacts::column_signs`), as a latency is: DDSketch's quantile +/// guarantee holds over it. +pub fn with_non_negative_values(metric: &str) -> DataWorkload { + DataWorkload { + sources: [( + metric.to_string(), + SourceFacts { + column_signs: [("value".to_string(), crate::stages::ValueSign::NonNegative)].into(), + ..SourceFacts::default() + }, + )] + .into(), + ..DataWorkload::default() + } +} + /// The state group of a sketch build of `algorithm` over `child` grouped /// by `reduction`, one build candidate per listed θ /// (`listed_build_parameters`). diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index cd05aa860..f23fb5384 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -185,10 +185,10 @@ pub struct SourceFacts { /// count separately: the bound is per output row. The deployment must /// enforce it; an observed distinct count is not a bound. It is for a /// distinct-count guarantee whose failure probability per output row - /// depends on the number of distinct items (classic HLL, a - /// distinct-count UnivMon); no guarantee in `asap_accuracy` reads it - /// yet, so those readouts serve no query with an accuracy target. KMV's - /// guarantee needs no bound. + /// depends on the number of distinct items: classic HLL's reads it (a + /// readout over sources without a bound serves no query with an + /// accuracy target); a distinct-count UnivMon has no guarantee yet. + /// KMV's guarantee needs no bound. #[serde(default)] pub max_distinct_per_evaluation: Option, /// How many distinct values each label (PromQL) or column (SQL) of the diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 079a5581e..c072a20c5 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -73,7 +73,7 @@ over a scope, a per-value failure probability `δ` becomes `V · δ` by the unio bound, without assuming independence, so a sketch shared by several groups (Hydra, a keyed Count-Min) is covered the same way (`AccuracyScope::max_values_covered` gives `V`). A guarantee whose own failure probability is -unknown (generic HLL) serves no query with an accuracy target +unknown (HLL without a distinct bound) serves no query with an accuracy target (`RootPruneReason::NoGuarantee`), and a top-k readout is admitted on its heap size and its value guarantee alone (the contracts table). @@ -182,8 +182,8 @@ The built-in models currently include: |---|---| | Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require error propagation | | KLL | Normalized rank error from the empirical 99% calibration `2.296 / k^0.9723`, fitted on Apache DataSketches' C++ KLL (`a9b42755072b`); below a failure probability of 0.01 that error is scaled by the Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`, a model rather than a measurement, and the guarantee names that contract apart | -| DDSketch | Relative value error from alpha under the supported estimator/domain contract; operators sensitive to sign or range require additional domain evidence | -| Generic HLL | RSE magnitude with unknown failure probability; not a general confidence theorem | +| DDSketch | Relative value error from alpha under the supported estimator/domain contract; a quantile's bound holds only over values of one sign (the state column's `column_signs` non-negative or non-positive), else the readout has no guarantee | +| Classic HLL | Relative error certified at the required failure probability over a source declaring at most `max_distinct_per_evaluation` distinct items (summed over the sources scanned below the state), from the linear-counting collision bound under independent uniform bucket hashes; without such a bound, or above the contract's range (`m/2` items at precision `p`), only the RSE magnitude with unknown failure probability | | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | | KMV | Provable distinct-count bound from the `k` smallest hashes | @@ -362,14 +362,19 @@ an observation into a guarantee. ### Example: HLL distinct counts -HLL's guarantee is its relative standard error, `1.04 / √m` for `m` buckets, -with an unknown failure probability. Its readout's metric is relative error, -but no failure probability can be evaluated, so a distinct count read from -HLL serves no query with an accuracy target -(`RootPruneReason::NoGuarantee`); KMV's distinct count, whose bound does not -depend on the number of distinct items, serves such a query. Missing evidence -leaves generic HLL confidence unknown. Invalid or infeasible contracts cannot -authorize the result. +HLL's generic guarantee is its relative standard error, `1.04 / √m` for `m` +buckets, with an unknown failure probability. Its readout's metric is relative +error, but no failure probability can be evaluated, so without more a distinct +count read from HLL serves no query with an accuracy target +(`RootPruneReason::NoGuarantee`). When the data workload bounds the distinct +items one evaluation counts (`SourceFacts::max_distinct_per_evaluation`, +summed over the sources scanned below the state), classic HLL's +linear-counting branch is certified: `COUNT(DISTINCT user_id)` within relative +0.05 at δ = 0.01 holds at precision 18 over at most 9,636 distinct users, and +over 1,000,000 (above the contract's `m/2`) it does not. KMV's distinct count, +whose bound does not depend on the number of distinct items, serves such a +query without a bound. Missing evidence leaves generic HLL confidence unknown. +Invalid or infeasible contracts cannot authorize the result. ## Sharing across consumers @@ -443,14 +448,17 @@ accuracy/src/ │ # the error contracts, each readout's required-accuracy │ # levels; MAX_ERROR_LEVELS ├── logical_edge/ # ErrorMetricsOfMemo::edge_meets_logical_requirements -│ # (α^log); within_limit, the requirement comparison +│ # (α^log), with DDSketch's sign condition; +│ # within_limit and tolerated_limit, the requirement +│ # comparison └── guarantee/ ├── mod.rs # theoretical_guarantee (one lookup at a failure │ # probability per algorithm) ├── binomial.rs # binomial_upper_tail ├── kll/ ├── ddsketch.rs - ├── hll.rs # HLL (unknown failure probability) + ├── hll/ # HLL: classic certificate over a distinct bound, else + │ # RSE (unknown failure probability) ├── cms/ ├── count_sketch/ ├── cardinality/ # KMV model From f3e7aab15f2f2692a5b230cce0d9fda4a1863549 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:34:45 +0000 Subject: [PATCH 2/4] refactor(stage3, accuracy): one composition rule for (K), ceilings in topological order, copies keyed on root limits, solver tolerance from the smallest limit; sign condition in the DDSketch guarantee (#1207 PR 8, review round 1) - AccuracyRules::compose is (K)'s rule once: selection_meets_targets and the ceilings call it; accuracy_constraints writes it as rows. - Ceilings in one pass in the memo's topological order (no fixpoint). - Context copies are keyed on each output's RootLimits, exactly what (K)'s root rows read; queries differing only in n_q without a whole-result delta are now copies. - HiGHS's feasibility tolerance is min(1e-7, 1e-3 x the smallest positive target limit), at least 1e-10, instead of a fixed 1e-10; scaling the rows by their limits kept the defaults but solved RangeHeavy-8users about 4x slower. - AccuracyTarget::failure_probability_per_output_value (renamed; None without a delta); readout_caps::query_levels is written on it. - CandidateForMemoGroup::required_accuracy, read by (K) and the report; readout_error_metric is crate-private again. - DDSketch's sign condition is in guarantee/ddsketch.rs (unknown failure probability over values of unknown sign), so theoretical_guarantee reads it too; ValueSign::is_one_sign; Stage 2 reads state_column_sign. - AccuracyMetric::COUNT and index; the distinct-bound fold moves to workload/bounds.rs. - accuracy-models.md: constraint (K) and Sign conditions sections. - Regression test: p90, p50 and p90 / p50 sharing readouts meet relative 0.05 only through (K). --- crates/accuracy/src/error_contract/mod.rs | 14 +- crates/accuracy/src/guarantee/cms/tests.rs | 2 +- .../src/guarantee/count_sketch/tests.rs | 1 + crates/accuracy/src/guarantee/ddsketch.rs | 25 +- crates/accuracy/src/guarantee/mod.rs | 34 +- crates/accuracy/src/guarantee/tests.rs | 54 +- crates/accuracy/src/lib.rs | 4 +- crates/accuracy/src/logical_edge/mod.rs | 70 +- crates/accuracy/src/logical_edge/tests.rs | 1 + crates/accuracy/src/metric_rule/mod.rs | 18 +- crates/accuracy/src/readout/mod.rs | 15 +- crates/accuracy/src/readout_caps/mod.rs | 17 +- .../tests/accuracy_at_query_roots.rs | 43 +- .../src/candidate.rs | 10 +- crates/stage3-plan-selection/src/lib.rs | 3 +- .../src/milp/accuracy.rs | 708 +++++++++++------- .../src/milp/equivalent_contexts.rs | 53 +- .../src/milp/model/mod.rs | 45 +- .../src/milp/model/tests.rs | 104 ++- crates/stage3-plan-selection/src/report.rs | 33 +- crates/types/src/accuracy/target/mod.rs | 40 +- crates/types/src/accuracy/target/tests.rs | 21 +- .../types/src/stages/logical/candidate/mod.rs | 15 +- .../stages/logical/group/column_properties.rs | 9 + crates/types/src/stages/logical/memo/mod.rs | 20 +- crates/types/src/workload/bounds.rs | 24 +- docs/design_docs/concepts/accuracy-models.md | 125 +++- 27 files changed, 995 insertions(+), 513 deletions(-) diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index ba76a5a56..c4517dd32 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -8,7 +8,7 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue}; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr}; -use asap_types::stages::{ColumnLineage, Memo, MemoGroupId, ValueSign}; +use asap_types::stages::{ColumnLineage, Memo, MemoGroupId}; use asap_types::summary_type::{ExactKind, SummaryInputExpr}; /// $g_c$'s linear bound for errors in `metric`, each input's and the @@ -221,13 +221,11 @@ fn copied_from_a_source( /// $|\sum_i (\hat{x}_i - x_i)| \le \varepsilon \sum_i |x_i| = /// \varepsilon |\sum_i x_i|$. Values of unknown sign may cancel. fn values_of_one_sign(memo: &Memo, group: MemoGroupId, column: usize) -> bool { - matches!( - memo.groups()[group.0 as usize] - .group_properties - .column_signs - .get(column), - Some(ValueSign::NonNegative | ValueSign::NonPositive) - ) + memo.groups()[group.0 as usize] + .group_properties + .column_signs + .get(column) + .is_some_and(|sign| sign.is_one_sign()) } /// Whether every key `reduction` groups `child`'s rows by is copied from a diff --git a/crates/accuracy/src/guarantee/cms/tests.rs b/crates/accuracy/src/guarantee/cms/tests.rs index 32379b26f..7087f2a0b 100644 --- a/crates/accuracy/src/guarantee/cms/tests.rs +++ b/crates/accuracy/src/guarantee/cms/tests.rs @@ -8,7 +8,7 @@ fn heap_evaluation_retains_frequency_metric() { heap_size: 10, }; let topk_frequency = - guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0, None) + guarantee_of_parameters(&cms_heap, &SketchStatistic::TopK { k: 10 }, 1.0, None, None) .expect("heap sketch still provides per-key frequency intervals"); assert_eq!( topk_frequency.spec.accuracy_metric, diff --git a/crates/accuracy/src/guarantee/count_sketch/tests.rs b/crates/accuracy/src/guarantee/count_sketch/tests.rs index 877015098..190df3858 100644 --- a/crates/accuracy/src/guarantee/count_sketch/tests.rs +++ b/crates/accuracy/src/guarantee/count_sketch/tests.rs @@ -37,6 +37,7 @@ fn count_sketch_uses_an_l2_guarantee() { }, 1.0, None, + None, ) .expect("CountSketch has a parameter-derived L2 guarantee"); assert_eq!( diff --git a/crates/accuracy/src/guarantee/ddsketch.rs b/crates/accuracy/src/guarantee/ddsketch.rs index fde3d390f..30cfdab75 100644 --- a/crates/accuracy/src/guarantee/ddsketch.rs +++ b/crates/accuracy/src/guarantee/ddsketch.rs @@ -1,13 +1,32 @@ -//! DDSketch relative value error under its estimator/domain contract. +//! DDSketch relative value error under its estimator/domain contract: +//! the bound holds only over values of one sign (`accuracy-models.md`, +//! "Sign conditions"), as the sketch's buckets around 0 have no relative bound. use super::*; -pub(super) fn guarantee(params: &SketchParams) -> Option { +/// DDSketch's guarantee at `params`: relative error α, failing never when +/// `state_sign`, the sign of the values the state was updated with, is +/// one sign ([`ValueSign::is_one_sign`]); else with an unknown failure +/// probability, so its readout serves no query with an accuracy target. +/// +/// Example: `quantile_over_time(0.99, latency[1h])` with `latency` +/// declared non-negative holds at α; over a gauge of unknown sign its +/// failure probability is unknown. +pub(super) fn guarantee( + params: &SketchParams, + state_sign: Option, +) -> Option { let SketchParams::DDSketch { alpha } = params else { return None; }; + let failure_probability = match state_sign.is_some_and(ValueSign::is_one_sign) { + true => ProbabilityExpr::Zero, + false => ProbabilityExpr::Unknown { + statistic: "ddsketch_values_of_unknown_sign".into(), + }, + }; Some(super::bounded_guarantee( AccuracyMetric::Relative, *alpha, - ProbabilityExpr::Zero, + failure_probability, )) } diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 0bd1fae3b..c26786881 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -1,6 +1,7 @@ //! Dispatch committed estimator parameters to their accuracy models. use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr}; use asap_types::ir::schema::FieldDataType; +use asap_types::stages::ValueSign; use asap_types::summary_type::{SketchParams, SketchStatistic}; use asap_types::summary_type::SummaryTypeCapability; @@ -25,12 +26,14 @@ use binomial::binomial_upper_tail; /// failure probability may exceed `failure_probability` or be unknown /// (HLL's, unless the source declares its distinct bound /// `max_distinct_per_evaluation`: classic HLL's is then certified at -/// `failure_probability`). A Hydra state reads each subpopulation's -/// state, so it has that state's guarantee. +/// `failure_probability`; DDSketch's, unless `state_sign`, the sign of the +/// values the state was updated with, is one sign). A Hydra state reads +/// each subpopulation's state, so it has that state's guarantee. fn guarantee_of_parameters( params: &SketchParams, query: &SketchStatistic, failure_probability: f64, + state_sign: Option, max_distinct_per_evaluation: Option, ) -> Option { match params { @@ -40,9 +43,10 @@ fn guarantee_of_parameters( &hydra.per_subpopulation(), query, failure_probability, + state_sign, max_distinct_per_evaluation, ), - SketchParams::DDSketch { .. } => ddsketch::guarantee(params), + SketchParams::DDSketch { .. } => ddsketch::guarantee(params, state_sign), SketchParams::Hll { .. } => { hll::guarantee(params, failure_probability, max_distinct_per_evaluation) } @@ -57,14 +61,19 @@ fn guarantee_of_parameters( /// The guarantee [`theoretical_guarantee`] gives, before it checks that θ /// fails at most `failure_probability` rarely: its metric, and whether its /// failure probability is known at all. `None` when the summary type does -/// not answer `statistic` or has no error model for it. -/// `max_distinct_per_evaluation` is the source's declared distinct bound -/// (`Memo::max_distinct_per_evaluation`), which a distinct count reads. +/// not answer `statistic` or has no error model for it. `state_sign` is +/// the sign of the values the read state was updated with +/// (`GroupProperties::state_column_sign`), which DDSketch's guarantee +/// reads; `max_distinct_per_evaluation` is the source's declared distinct +/// bound (`Memo::max_distinct_per_evaluation`), which a distinct count +/// reads. Without them, the guarantee's metric still holds, with an +/// unknown failure probability. pub(crate) fn guarantee_before_failure_probability_check( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, failure_probability: f64, + state_sign: Option, max_distinct_per_evaluation: Option, ) -> Option { if !matches!(summary_type, FieldDataType::Sketch(_)) @@ -76,6 +85,7 @@ pub(crate) fn guarantee_before_failure_probability_check( parameters, statistic, failure_probability, + state_sign, max_distinct_per_evaluation, ) } @@ -105,15 +115,18 @@ fn bounded_guarantee( /// that rarely. `None` when this θ cannot fail that rarely, when the /// summary type does not answer `statistic` (its capability row) or has no /// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch -/// readouts without a certified bound). A distinct count from classic HLL -/// is certified only over a source of at most `max_distinct_per_evaluation` -/// distinct items (`Memo::max_distinct_per_evaluation`); `None` without that -/// bound. +/// readouts without a certified bound). A quantile from DDSketch is +/// certified only over values of one sign, `state_sign` +/// (`GroupProperties::state_column_sign`; `accuracy-models.md`, "Sign +/// conditions"), and a distinct count from classic HLL only over a source +/// of at most `max_distinct_per_evaluation` distinct items +/// (`Memo::max_distinct_per_evaluation`); `None` without them. pub fn theoretical_guarantee( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, failure_probability: f64, + state_sign: Option, max_distinct_per_evaluation: Option, ) -> Option { guarantee_before_failure_probability_check( @@ -121,6 +134,7 @@ pub fn theoretical_guarantee( parameters, statistic, failure_probability, + state_sign, max_distinct_per_evaluation, ) .filter(|guarantee| holds_at_failure_probability(guarantee, failure_probability)) diff --git a/crates/accuracy/src/guarantee/tests.rs b/crates/accuracy/src/guarantee/tests.rs index ea98d238e..370487a68 100644 --- a/crates/accuracy/src/guarantee/tests.rs +++ b/crates/accuracy/src/guarantee/tests.rs @@ -2,6 +2,10 @@ use super::*; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::summary_type::SketchAlgorithm; +/// Values of one sign, which every listed guarantee holds over (DDSketch's +/// needs it). +const ONE_SIGN: Option = Some(ValueSign::NonNegative); + /// Every listed θ has a theoretical guarantee for the statistic its /// algorithm is listed for, at failure probability 0.01, in the /// algorithm's metric; along a list of one structure the error never @@ -49,9 +53,15 @@ fn every_listed_parameters_has_a_guarantee_at_one_percent() { let errors: Vec = parameter_list(algorithm) .iter() .map(|parameters| { - let guarantee = - theoretical_guarantee(&summary_type, parameters, &statistic, 0.01, None) - .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); + let guarantee = theoretical_guarantee( + &summary_type, + parameters, + &statistic, + 0.01, + ONE_SIGN, + None, + ) + .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); assert_eq!(guarantee.spec.accuracy_metric, metric, "{algorithm:?}"); guarantee.spec.error.evaluate().unwrap() }) @@ -78,7 +88,7 @@ fn a_guarantee_is_evaluated_at_the_required_failure_probability() { let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); let k = SketchParams::Kll { k: 547 }; let error_at = |delta| { - theoretical_guarantee(&kll, &k, &quantile, delta, None) + theoretical_guarantee(&kll, &k, &quantile, delta, None, None) .unwrap() .spec .error @@ -93,9 +103,37 @@ fn a_guarantee_is_evaluated_at_the_required_failure_probability() { value: None, }; let depth = |depth| SketchParams::Cms { width: 272, depth }; - assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2, None).is_some()); - assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4, None).is_none()); - assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4, None).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2, None, None).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4, None, None).is_none()); + assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4, None, None).is_some()); // A statistic the summary type does not answer has no guarantee. - assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01, None).is_none()); + assert!( + theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01, None, None).is_none() + ); +} + +/// DDSketch's relative error on a quantile holds only over values of one +/// sign: over non-negative or non-positive values at α, at any failure +/// probability; over values of unknown sign, or of no known sign, the +/// guarantee is not certified at any failure probability, though its +/// metric is still relative error. +#[test] +fn a_ddsketch_quantile_is_certified_only_over_values_of_one_sign() { + let ddsketch = FieldDataType::Sketch(SketchAlgorithm::DDSketch); + let alpha = SketchParams::DDSketch { alpha: 0.01 }; + let quantile = SketchStatistic::Quantile { q: 0.99 }; + for (sign, certified) in [ + (Some(ValueSign::NonNegative), true), + (Some(ValueSign::NonPositive), true), + (Some(ValueSign::Unknown), false), + (None, false), + ] { + let theoretical = theoretical_guarantee(&ddsketch, &alpha, &quantile, 0.01, sign, None); + assert_eq!(theoretical.is_some(), certified, "{sign:?}"); + let before_check = guarantee_before_failure_probability_check( + &ddsketch, &alpha, &quantile, 0.01, sign, None, + ) + .unwrap(); + assert_eq!(before_check.spec.accuracy_metric, AccuracyMetric::Relative); + } } diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index f335e77c3..3b049a6f9 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -6,8 +6,7 @@ //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the //! metrics each query root allows, [`query_roots_with_target`], and the -//! metric of a readout's error, [`readout_error_metric`], from `readout`), -//! each readout's required +//! metric of a readout's error from `readout`), each readout's required //! accuracy ([`readout_levels`], in `readout_caps`), and $\alpha^{\log}$ of //! a logical edge ([`ErrorMetricsOfMemo::edge_meets_logical_requirements`], //! in `logical_edge`, with a bound's comparison to a limit @@ -30,5 +29,4 @@ pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::{tolerated_limit, within_limit}; pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; -pub use readout::readout_error_metric; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/logical_edge/mod.rs b/crates/accuracy/src/logical_edge/mod.rs index beb50cc51..e72cdb8e3 100644 --- a/crates/accuracy/src/logical_edge/mod.rs +++ b/crates/accuracy/src/logical_edge/mod.rs @@ -4,12 +4,9 @@ //! ([`within_limit`]). use asap_types::accuracy::AccuracyMetric; -use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; -use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason, ValueSign}; -use asap_types::summary_type::{ - MergedError, SketchAlgorithm, SketchParams, SketchStatistic, SummaryTypeCapability, -}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; +use asap_types::summary_type::{MergedError, SketchParams, SketchStatistic, SummaryTypeCapability}; use crate::guarantee::{guarantee_before_failure_probability_check, holds_at_failure_probability}; use crate::readout::sketch_readout; @@ -49,9 +46,9 @@ impl ErrorMetricsOfMemo { /// guarantee of the producer's θ for the readout's statistic, at the /// required failure probability, is within the required error /// ([`within_limit`]) and in its metric: the deployment's measurement - /// (`evidence`) when it has one, else the theoretical guarantee when - /// its sign condition holds for the state's values - /// ([`sign_condition_holds`]), a distinct count's over the scanned + /// (`evidence`) when it has one, else the theoretical guarantee over + /// the sign of the state's values (`GroupProperties::state_column_sign`, + /// DDSketch's sign condition) and, for a distinct count, the scanned /// sources' declared distinct bound /// (`Memo::max_distinct_per_evaluation`); a /// merged state (any producer but a build) must keep the guarantee @@ -118,25 +115,23 @@ impl ErrorMetricsOfMemo { let measured = evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); let allowed = self.error_metrics_allowed_on_input_edge(consumer, input); - // The sign of the values the read states were updated with. - let state_sign = memo.groups()[summary_input.0 as usize] - .group_properties - .state_column_sign(); // The theoretical guarantee before its failure-probability check, - // computed once for both checks below. - let theoretical = (measured.is_none() - && (allowed.is_some() || required.is_some()) - && sign_condition_holds(summary_type, statistic, state_sign)) - .then(|| { - guarantee_before_failure_probability_check( - summary_type, - parameters, - statistic, - failure_probability, - memo.max_distinct_per_evaluation(summary_input), - ) - }) - .flatten(); + // computed once for both checks below, over the sign of the values + // the read states were updated with. + let theoretical = (measured.is_none() && (allowed.is_some() || required.is_some())) + .then(|| { + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + failure_probability, + memo.groups()[summary_input.0 as usize] + .group_properties + .state_column_sign(), + memo.max_distinct_per_evaluation(summary_input), + ) + }) + .flatten(); if let Some(allowed) = allowed { // A query with a target reads this readout: its error needs a // metric and a failure probability a guarantee states. @@ -190,29 +185,6 @@ impl ErrorMetricsOfMemo { } } -/// Whether the theoretical guarantee of reading `statistic` from a state -/// of `summary_type` holds for states updated with values of sign `sign` -/// (`GroupProperties::state_column_sign`; concepts doc, section 2, "Sign -/// conditions"): DDSketch's relative error on a quantile needs values of -/// one sign, `NonNegative` or `NonPositive`; every other guarantee holds -/// whatever their sign. -/// -/// Example: `quantile_over_time(0.99, latency[1h])` from DDSketch with -/// `latency` declared non-negative holds; over a gauge of unknown sign it -/// does not, as the sketch's buckets around 0 have no relative bound. -fn sign_condition_holds( - summary_type: &FieldDataType, - statistic: &SketchStatistic, - sign: Option, -) -> bool { - match (summary_type, statistic) { - (FieldDataType::Sketch(SketchAlgorithm::DDSketch), SketchStatistic::Quantile { .. }) => { - matches!(sign, Some(ValueSign::NonNegative | ValueSign::NonPositive)) - } - _ => true, - } -} - /// A heap variant's heap size. fn heap_size(parameters: &SketchParams) -> Option { match parameters { diff --git a/crates/accuracy/src/logical_edge/tests.rs b/crates/accuracy/src/logical_edge/tests.rs index bf44659a7..7f31c343b 100644 --- a/crates/accuracy/src/logical_edge/tests.rs +++ b/crates/accuracy/src/logical_edge/tests.rs @@ -6,6 +6,7 @@ use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; use asap_types::stages::{ CandidateForMemoGroup, LogicalRequirements, OriginOfCandidateForMemoGroup, RequiredAccuracy, + ValueSign, }; use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_builders::query; diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index f176c2229..6e947e0e4 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -15,12 +15,9 @@ use crate::error_contract::{error_contract, ErrorContract}; use crate::readout::sketch_readout_error_metric; use crate::readout_caps::caps; -/// How many error metrics there are: `AccuracyMetric::ALL`'s length. -pub(crate) const METRIC_COUNT: usize = AccuracyMetric::ALL.len(); - /// Per metric, in `AccuracyMetric::ALL` order: a candidate's error /// contract in it; `None` = no contract. -pub(crate) type ErrorContractPerMetric = [Option; METRIC_COUNT]; +pub(crate) type ErrorContractPerMetric = [Option; AccuracyMetric::COUNT]; /// The error metrics of one memo state, built once per memo state before /// an edge pass; Stage 3 reads the same contracts. @@ -103,9 +100,8 @@ impl ErrorMetricsOfMemo { candidate: CandidateForMemoGroupId, metric: AccuracyMetric, ) -> Option<&ErrorContract> { - self.error_contract[candidate.group.0 as usize][candidate.index_in_group] - [metric_index(metric)] - .as_ref() + self.error_contract[candidate.group.0 as usize][candidate.index_in_group][metric.index()] + .as_ref() } /// The error metric of `candidate` when it is a sketch readout. @@ -272,14 +268,6 @@ impl ErrorMetricsOfMemo { } } -/// The index of `metric` in `AccuracyMetric::ALL`. -fn metric_index(metric: AccuracyMetric) -> usize { - AccuracyMetric::ALL - .iter() - .position(|&listed| listed == metric) - .expect("AccuracyMetric::ALL lists every metric") -} - /// Each query with a target, with the groups its root reads and the metrics /// each allows: an operator root, `scalar(v)` and an SQL scalar subquery /// (which return the value they read) allow the target's metrics diff --git a/crates/accuracy/src/readout/mod.rs b/crates/accuracy/src/readout/mod.rs index 4ebc1d69b..7e599e082 100644 --- a/crates/accuracy/src/readout/mod.rs +++ b/crates/accuracy/src/readout/mod.rs @@ -15,7 +15,7 @@ use crate::guarantee::guarantee_before_failure_probability_check; /// error although its failure probability is unknown). `None` for an exact /// state and for a summary type with no listed θ or no guarantee for the /// statistic (UnivMon's distinct count). -pub fn readout_error_metric( +pub(crate) fn readout_error_metric( summary_type: &FieldDataType, statistic: &SketchStatistic, ) -> Option { @@ -24,9 +24,16 @@ pub fn readout_error_metric( }; let parameters = parameter_list(*algorithm).first()?; Some( - guarantee_before_failure_probability_check(summary_type, parameters, statistic, 1.0, None)? - .spec - .accuracy_metric, + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + 1.0, + None, + None, + )? + .spec + .accuracy_metric, ) } diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index d4bb4c854..3d3016bf3 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -169,10 +169,10 @@ fn group_levels( /// depend on does, so by the union bound δ splits over all of them, /// `readout_values` ([`ReadoutsReadByQueries::readout_values`], summed over /// every readout group below the query), and over the values its scope -/// covers, $V$ (`AccuracyScope::max_values_covered`, $n_q$ for a whole -/// result): the per-value failure probability is -/// $\delta / \max(V, \text{readout values})$, and needs both known when the -/// target sets a δ. +/// covers, $V$ (`AccuracyTarget::failure_probability_per_output_value`, +/// $\delta / V$ with $V = n_q$ for a whole result): the per-value failure +/// probability is $\min(\delta / V, \delta / \text{readout values})$, and +/// needs both known when the target sets a δ. /// - `max(quantile_over_time(0.9, m[5m]))` over 1000 series at δ = 0.01 /// returns one value but reads 1000 readout values: 1e-5. /// - `max by (job) (q{host="h0"} or q{host="h1"})` with 3 series per side @@ -192,12 +192,9 @@ fn query_levels( ) -> Option> { let per_value_failure_probability = match target.failure_probability { None => 1.0, - Some(delta) => { - let values_covered = target - .accuracy_scope - .max_values_covered(number_of_values_per_query_evaluation)?; - delta / values_covered.max(readout_values?).max(1) as f64 - } + Some(delta) => target + .failure_probability_per_output_value(number_of_values_per_query_evaluation)? + .min(delta / readout_values?.max(1) as f64), }; let level = |max_error, failure_probability| RequiredAccuracy { metric, diff --git a/crates/integration-tests/tests/accuracy_at_query_roots.rs b/crates/integration-tests/tests/accuracy_at_query_roots.rs index 2316e3c1f..3ef77fac5 100644 --- a/crates/integration-tests/tests/accuracy_at_query_roots.rs +++ b/crates/integration-tests/tests/accuracy_at_query_roots.rs @@ -1,7 +1,7 @@ //! Constraint (K), each query's composed accuracy at its root, through -//! every planning stage on #1207's problem example (concepts doc, -//! section 1): A, the p95 of `latency` at rank error 0.01, and B, the -//! ratio of its p99 and p50 at relative error 0.03. +//! every planning stage on a two-query example (`accuracy-models.md`, +//! "Example: allocation in a ratio"): A, the p95 of `latency` at rank +//! error 0.01, and B, the ratio of its p99 and p50 at relative error 0.03. use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; use asap_planner::{DeploymentCapabilities, PlanningModels}; @@ -62,7 +62,7 @@ fn selection(queries: &[(&str, &str)], ignore_accuracy: bool) -> (PhysicalASAPDA (plan.clone(), accepted) } -/// #1207's problem example. +/// Queries A and B of the module's example. const PROBLEM_EXAMPLE: [(&str, &str); 2] = [ ( "quantile_over_time(0.95, latency[1h])", @@ -153,7 +153,8 @@ fn ignoring_accuracy_plans_without_k() { /// A whole-result failure probability of 0.01 over 1,000 series is 1e-5 /// per value: a KLL read at that failure probability meets rank error -/// 0.01 only from k = 547 up (the concepts doc's KLL table). +/// 0.01 only from k = 547 up (`accuracy-models.md`, the KLL row of the +/// built-in models: 2.296 / k^0.9723 scaled by the Gaussian quantile ratio). #[test] fn a_whole_result_failure_probability_is_divided_over_the_values() { let plan = plan( @@ -172,3 +173,35 @@ fn a_whole_result_failure_probability_is_divided_over_the_values() { ); } } + +/// p90, p50 and p90 / p50 over `latency[15m]`, each at relative 0.05: the +/// two quantiles alone accept DDSketch readouts at α = 0.05, and sharing +/// those with the ratio is cheapest, but the ratio's composed bound would be +/// (0.05 + 0.05) / (1 − 0.1) = 0.111. (K) is what rules that out: the plan +/// reads readouts at α ≤ 0.02 for the ratio, (0.02 + 0.02) / 0.9 = 0.044. +#[test] +fn a_ratio_sharing_its_readouts_with_its_operands_still_meets_its_target() { + let target = r#"{"accuracy_metric": "relative", "max_error": 0.05}"#; + let plan = plan( + &[ + ("quantile_over_time(0.9, latency[15m])", target), + ("quantile_over_time(0.5, latency[15m])", target), + ( + "quantile_over_time(0.9, latency[15m]) / quantile_over_time(0.5, latency[15m])", + target, + ), + ], + false, + ); + let alphas: Vec = sketch_parameters(&plan) + .iter() + .filter_map(|p| match p { + SketchParams::DDSketch { alpha } => Some(*alpha), + _ => None, + }) + .collect(); + assert!( + alphas.iter().any(|&alpha| alpha <= 0.02), + "the ratio reads readouts composing within 0.05: {alphas:?}" + ); +} diff --git a/crates/stage2-physical-optimization/src/candidate.rs b/crates/stage2-physical-optimization/src/candidate.rs index 86d658ebc..d58b0f928 100644 --- a/crates/stage2-physical-optimization/src/candidate.rs +++ b/crates/stage2-physical-optimization/src/candidate.rs @@ -270,12 +270,8 @@ fn count_min_weights_not_proven_non_negative( ) { return false; } - let properties = &memo.groups()[group.0 as usize].group_properties; - let weight_sign = properties - .schema - .fields - .iter() - .position(|field| !field.is_plain()) - .map(|state| properties.column_signs[state]); + let weight_sign = memo.groups()[group.0 as usize] + .group_properties + .state_column_sign(); weight_sign != Some(ValueSign::NonNegative) } diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 87d77d5a8..e00aa207c 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -18,7 +18,8 @@ //! builds the plan (`PhysicalASAPDAG`) from the selected executions and //! edges as its last step ([`plan`], each node retaining its candidate's //! $h_v$, the needs from [`retention`]). Accuracy at each query root is -//! constraint (K) (section 4): error bounds $E_{\kappa,c,m}$ per execution +//! constraint (K) (`accuracy-models.md`, "Checking each query's target in +//! selection"): error bounds $E_{\kappa,c,m}$ per execution //! and metric and failure probabilities $D_{\kappa,c}$, composed along the //! selected edges and checked at each query's root. Costs come from //! `asap-cost`. diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index 6d41ba838..80827aa9e 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -1,30 +1,76 @@ //! The accuracy extension of the selection MILP -//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 4, "Query composition and root checks"): constraint (K), per -//! execution and per error metric, over the error each execution adds and -//! passes on ([`AccuracyRules`], [`ErrorModel`]), and the same rule -//! checked on one selection ([`AccuracyRules::selection_meets_targets`]), -//! which the greedy upper bound needs to be a feasible plan of (K). +//! (`docs/design_docs/concepts/accuracy-models.md`, "Checking each query's +//! target in selection: constraint (K)"): constraint (K), per execution and +//! per error metric, over the error each execution adds and passes on +//! ([`AccuracyRules`], [`ErrorModel`]), and the same rule checked on one +//! selection ([`AccuracyRules::selection_meets_targets`]), which the greedy +//! upper bound needs to be a feasible plan of (K). One composition rule, +//! [`AccuracyRules::compose`], gives both that check and the ceilings of +//! (K)'s big-M rows; [`accuracy_constraints`] writes the same rule as rows. use std::collections::{BTreeMap, HashMap}; +use std::hash::{Hash, Hasher}; use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; use asap_accuracy::{query_roots_with_target, tolerated_limit, ErrorContract, ErrorMetricsOfMemo}; use asap_types::accuracy::AccuracyMetric; -use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId, RequiredAccuracy}; +use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId}; use super::equivalent_contexts::SharedVariables; use super::model::SelectionInputs; use crate::selectable::QueryId; -/// The number of error metrics, `AccuracyMetric::ALL`'s length. -const METRIC_COUNT: usize = AccuracyMetric::ALL.len(); +/// An execution's error as (K) composes it: its error bound per metric +/// (by [`AccuracyMetric::index`]) and its failure probability. +/// +/// Example: a DDSketch readout required at relative 0.01 with δ = 1e-3 +/// is 0.01 in `Relative`, 0 in every other metric, and 1e-3. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub(super) struct ExecutionError { + /// $E_{c,m}$ per metric. + error_bound_per_metric: [f64; AccuracyMetric::COUNT], + /// $D_c$. + failure_probability: f64, +} + +impl ExecutionError { + /// The larger of `self` and `other` in each metric and in failure + /// probability, as the most any one of several producers passes on. + fn largest(self, other: Self) -> Self { + let mut largest = self; + for (error, other) in largest + .error_bound_per_metric + .iter_mut() + .zip(other.error_bound_per_metric) + { + *error = error.max(other); + } + largest.failure_probability = largest.failure_probability.max(other.failure_probability); + largest + } + + /// Whether it is 0 in every metric and in failure probability. + fn is_zero(&self) -> bool { + self.failure_probability == 0.0 && self.error_bound_per_metric.iter().all(|&e| e == 0.0) + } +} + +/// What composing one execution's error from what its inputs read gives +/// ([`AccuracyRules::compose`]). +struct Composition { + /// Its own error plus each input's error times the contract's factor, + /// and its own failure probability plus each input's. + error: ExecutionError, + /// Whether every input's error passes the execution's metric rule (it + /// has a contract in that metric) and the contract's validity cap. + inputs_admitted: bool, +} /// What (K) reads of the memo, built once per selection when some query -/// has an accuracy target: each candidate's error contracts, which -/// executions a query with a target reaches, and what each query root -/// allows. +/// has an accuracy target: each candidate's error contracts, each +/// execution's own error, which executions a query with a target reaches, +/// and what each query root allows. pub(super) struct AccuracyRules { /// The candidates' error contracts per metric /// (`ErrorMetricsOfMemo::error_contract`). @@ -32,24 +78,82 @@ pub(super) struct AccuracyRules { /// Per execution, by position in `SelectableExecutions::executions`, /// its logical candidate. candidate_of_execution: Vec, + /// Per execution, the error it adds: a readout's requirement + /// (`CandidateForMemoGroup::required_accuracy`), 0 for any other. + own_error_per_execution: Vec, /// Per execution, whether a query with an accuracy target reaches its /// group. - targeted: Vec, + reached_by_query_with_target: Vec, + /// Every execution, its producers' groups before its own (the memo's + /// topological order): an order that composes each producer before + /// its consumers. + executions_producers_first: Vec, /// Per (query, output group its root reads), what its root check /// bounds the output by. - roots: HashMap<(usize, MemoGroupId), RootLimits>, + roots: HashMap<(QueryId, MemoGroupId), RootLimits>, + /// The smallest positive limit a target sets (an ε, a δ per output + /// value), before [`tolerated_limit`] widens it; `None` when none does + /// (exact targets only). The solver's feasibility tolerance is set + /// from it ([`Self::feasibility_tolerance`]). + smallest_positive_limit: Option, +} + +/// What a root check of (K) bounds one query output by, each limit +/// already widened by [`tolerated_limit`] as the edge check widens it. +/// Two outputs with equal limits have the same root rows, so they compare +/// and hash by the limits' bits. +/// +/// Example: `p99 / p50` at relative 0.03 with δ = 0.01 over a whole result +/// of 100 values allows 0.03 (widened) in `Relative`, 0 in every other +/// metric, and 1e-4 (widened) failure probability per output value. +#[derive(Debug, Clone, Copy)] +pub(super) struct RootLimits { + /// Per metric (by [`AccuracyMetric::index`]), the error the output may + /// have: the target's ε in a metric the root allows + /// ([`query_roots_with_target`]), 0 in any other. + tolerated_error_per_metric: [f64; AccuracyMetric::COUNT], + /// The failure probability each output value may have + /// (`AccuracyTarget::failure_probability_per_output_value`); `None` + /// without a δ, or with one over the whole result and $n_q$ unknown + /// (Pass 2 then gives that query's readouts no level). + tolerated_failure_probability_per_output_value: Option, +} + +impl RootLimits { + /// The limits' bits, which equality, hashing and order compare. + fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option) { + ( + self.tolerated_error_per_metric.map(f64::to_bits), + self.tolerated_failure_probability_per_output_value + .map(f64::to_bits), + ) + } +} + +impl PartialEq for RootLimits { + fn eq(&self, other: &Self) -> bool { + self.bits() == other.bits() + } +} + +impl Eq for RootLimits {} + +impl Hash for RootLimits { + fn hash(&self, state: &mut H) { + self.bits().hash(state); + } +} + +impl PartialOrd for RootLimits { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } } -/// What a root check of (K) bounds one query output by. -struct RootLimits { - /// The target's ε. - max_error: f64, - /// The metrics allowed at the root ([`query_roots_with_target`]). - allowed_metrics: Vec, - /// $\delta_q$ per value (`AccuracyTarget::failure_probability_per_value`); - /// `None` without a δ, or with one over the whole result and $n_q$ - /// unknown (Pass 2 gives that query's readouts no level). - failure_probability_per_value: Option, +impl Ord for RootLimits { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.bits().cmp(&other.bits()) + } } impl AccuracyRules { @@ -77,7 +181,18 @@ impl AccuracyRules { physical.physical_candidates[execution.candidate.0].logical_candidate_id }) .collect(); - let targeted = candidate_of_execution + let own_error_per_execution = candidate_of_execution + .iter() + .map(|&id| { + let mut own = ExecutionError::default(); + if let Some(required) = memo.candidate(id).required_accuracy() { + own.error_bound_per_metric[required.metric.index()] = required.max_error; + own.failure_probability = required.failure_probability; + } + own + }) + .collect(); + let reached_by_query_with_target = candidate_of_execution .iter() .map(|id| { memo.groups()[id.group.0 as usize] @@ -86,45 +201,144 @@ impl AccuracyRules { .any(|&query| memo.queries()[query].accuracy_target.is_some()) }) .collect(); + let mut position_in_order = vec![0; memo.groups().len()]; + for (position, group) in order.iter().enumerate() { + position_in_order[group.0 as usize] = position; + } + let mut executions_producers_first: Vec = + (0..candidate_of_execution.len()).collect(); + executions_producers_first + .sort_by_key(|&c| position_in_order[candidate_of_execution[c].group.0 as usize]); + let mut smallest_positive_limit: Option = None; + let mut see_limit = |limit: f64| { + if limit > 0.0 { + smallest_positive_limit = + Some(smallest_positive_limit.map_or(limit, |smallest| smallest.min(limit))); + } + }; let roots = query_roots_with_target(memo) .into_iter() .map(|(query, root, target, allowed_metrics)| { - let failure_probability_per_value = - target - .failure_probability - .and(target.failure_probability_per_value( - memo.queries()[query].number_of_values_per_query_evaluation, - )); + let failure_probability = target.failure_probability_per_output_value( + memo.queries()[query].number_of_values_per_query_evaluation, + ); + let mut tolerated_error_per_metric = [0.0; AccuracyMetric::COUNT]; + for metric in allowed_metrics { + tolerated_error_per_metric[metric.index()] = tolerated_limit(target.max_error); + see_limit(target.max_error); + } + if let Some(limit) = failure_probability { + see_limit(limit); + } let limits = RootLimits { - max_error: target.max_error, - allowed_metrics, - failure_probability_per_value, + tolerated_error_per_metric, + tolerated_failure_probability_per_output_value: failure_probability + .map(tolerated_limit), }; - ((query, root), limits) + ((QueryId(query), root), limits) }) .collect(); Some(Self { error_metrics: ErrorMetricsOfMemo::of(memo, &order), candidate_of_execution, - targeted, + own_error_per_execution, + reached_by_query_with_target, + executions_producers_first, roots, + smallest_positive_limit, }) } - /// Execution `c`'s error contract in metric `m` (by position in - /// `AccuracyMetric::ALL`); `None` where it has none (the metric rule). + /// The primal and MIP feasibility tolerance HiGHS solves (K) at: + /// $\min(10^{-7}, 10^{-3} \cdot \ell)$ for the smallest positive limit + /// $\ell$ a target sets, so a row may pass its limit by at most a + /// thousandth of the smallest one, but not below 1e-10, the least + /// HiGHS accepts (a per-value δ under 1e-7 is then checked only to + /// 1e-10); `None` (HiGHS's defaults) when it is not below 1e-7. + /// HiGHS's tolerances are absolute: δ = 0.01 over a whole result of 1e4 + /// values is 1e-6 per value, of which the default 1e-7 is a tenth. + /// Scaling (K)'s rows by their limits instead kept the defaults but + /// solved RangeHeavy-8users about 4× slower (larger big-M + /// coefficients). + /// + /// Example: relative 0.05 with δ = 0.01 over 1e4 values: 1e-9. + pub(super) fn feasibility_tolerance(&self) -> Option { + let tolerance = (1e-3 * self.smallest_positive_limit?).max(1e-10); + (tolerance < 1e-7).then_some(tolerance) + } + + /// What the root check of `query`'s output reading `group` bounds it + /// by; `None` when the query has no target, so (K) checks nothing + /// there. + pub(super) fn root_limits(&self, query: QueryId, group: MemoGroupId) -> Option<&RootLimits> { + self.roots.get(&(query, group)) + } + + /// Execution `c`'s error contract in metric `m` (by + /// [`AccuracyMetric::index`]); `None` where it has none (the metric + /// rule). fn contract(&self, c: usize, m: usize) -> Option<&ErrorContract> { self.error_metrics .error_contract(self.candidate_of_execution[c], AccuracyMetric::ALL[m]) } + /// Execution `c`'s error composed from `inputs_read`, the error read on + /// each of its inputs (input index, error), by (K)'s rule + /// (`accuracy-models.md`, "Checking each query's target in + /// selection"): its own error + /// plus, per metric, $\beta_{c,i,m}$ times each input's error in that + /// metric where the input's error reaches the output ($\beta > 0$), + /// and its own failure probability plus every input's (a union bound). + /// An input's error in a metric `c` has no contract for breaks the + /// metric rule, and one above the contract's validity cap $b_{c,i}$ + /// breaks the cap: either leaves the inputs not admitted, and adds + /// nothing in that metric for the metric rule. + /// + /// Example: `p99 / p50`'s contract passes each operand's relative + /// error with factor 1.111 and caps the denominator at 0.1: reading + /// 0.02 and 0.02 composes 0.044, admitted; reading a denominator at + /// 0.2 is not admitted. + fn compose( + &self, + c: usize, + inputs_read: impl IntoIterator, + ) -> Composition { + let mut composed = Composition { + error: self.own_error_per_execution[c], + inputs_admitted: true, + }; + for (i, read) in inputs_read { + composed.error.failure_probability += read.failure_probability; + for (m, &error) in read.error_bound_per_metric.iter().enumerate() { + if error <= 0.0 { + continue; + } + let Some(contract) = self.contract(c, m) else { + composed.inputs_admitted = false; + continue; + }; + if !contract.input_error_reaches_output(i) { + continue; + } + if contract.linear_bound_holds_up_to_per_input[i] + .is_some_and(|holds_up_to| error > tolerated_limit(holds_up_to)) + { + composed.inputs_admitted = false; + } + composed.error.error_bound_per_metric[m] += + contract.error_factor_per_input[i] * error; + } + } + composed + } + /// Whether the selection `edges` (by position in /// `SelectableExecutions::edges`) with the outputs `answering` (query, /// execution answering it, in `SelectableExecutions::answering` order) - /// meets (K): the composed error of each execution a query with a - /// target reaches, by the rule of [`accuracy_constraints`] on the - /// selected edges, passes its consumers' metric rule and validity caps - /// and, at each output of a query with a target, the root's limits. + /// meets (K): each execution a query with a target reaches, composed + /// ([`Self::compose`]) from the producers selected on its inputs, + /// admits its inputs' errors and, at each output of a query with a + /// target, is within the root's limits. /// /// Example: p99 / p50 at relative 0.03 over two readouts at 0.02 does /// not meet it (1.111 · 0.04 = 0.044); over two at 0.01 it does. @@ -142,8 +356,7 @@ impl AccuracyRules { let input = inputs.physical.edge(edge.edge).input_index_of_consumer; reads[edge.consumer].push((input, edge.producer)); } - // Composed (E per metric, D) per execution, producers first. - let mut composed: Vec> = vec![None; count]; + let mut composed: Vec> = vec![None; count]; let mut on_walk = vec![false; count]; let mut pending: Vec<(usize, bool)> = answering.iter().map(|&(_, v)| (v, false)).collect(); while let Some((c, producers_done)) = pending.pop() { @@ -165,57 +378,42 @@ impl AccuracyRules { ); continue; } - if !self.targeted[c] { + if !self.reached_by_query_with_target[c] { // No query with a target reaches it: (K) has no rows for // it, nor for its producers. - composed[c] = Some(([0.0; METRIC_COUNT], 0.0)); + composed[c] = Some(ExecutionError::default()); continue; } - let (mut error_bound, mut failure_probability) = own_error_of(own_error(inputs, c)); - for &(i, u) in &reads[c] { - let Some((producer_error, producer_failure)) = composed[u] else { - return false; - }; - failure_probability += producer_failure; - for (m, &error) in producer_error.iter().enumerate() { - if error <= 0.0 { - continue; - } - let Some(contract) = self.contract(c, m) else { - return false; - }; - if !contract.input_error_reaches_output(i) { - continue; - } - if contract.linear_bound_holds_up_to_per_input[i] - .is_some_and(|holds_up_to| error > tolerated_limit(holds_up_to)) - { - return false; - } - error_bound[m] += contract.error_factor_per_input[i] * error; - } + let Some(inputs_read) = reads[c] + .iter() + .map(|&(i, u)| Some((i, composed[u]?))) + .collect::>>() + else { + return false; + }; + let composition = self.compose(c, inputs_read); + if !composition.inputs_admitted { + return false; } - composed[c] = Some((error_bound, failure_probability)); + composed[c] = Some(composition.error); } let memo = &inputs.physical.logical_candidates.memo; - for (&(query, group), &(_, v)) in memo.output_groups().iter().zip(answering) { - let Some(limits) = self.roots.get(&(query, group)) else { + for (&(_, group), &(query, v)) in memo.output_groups().iter().zip(answering) { + let Some(limits) = self.root_limits(query, group) else { continue; }; - let (error_bound, failure_probability) = - composed[v].expect("each answering execution is composed"); - for (m, &error) in error_bound.iter().enumerate() { - let limit = match limits.allowed_metrics.contains(&AccuracyMetric::ALL[m]) { - true => tolerated_limit(limits.max_error), - false => 0.0, - }; - if error > limit { - return false; - } + let error = composed[v].expect("each answering execution is composed"); + if error + .error_bound_per_metric + .iter() + .zip(limits.tolerated_error_per_metric) + .any(|(&error, limit)| error > limit) + { + return false; } if limits - .failure_probability_per_value - .is_some_and(|limit| failure_probability > tolerated_limit(limit)) + .tolerated_failure_probability_per_output_value + .is_some_and(|limit| error.failure_probability > limit) { return false; } @@ -224,37 +422,6 @@ impl AccuracyRules { } } -/// $(\varepsilon_c, \delta_c)$ in metric $m_c$ of the execution at -/// position `index`: a readout's requirement, the error it adds; `None` for -/// any other candidate. -fn own_error(inputs: &SelectionInputs<'_>, index: usize) -> Option { - let physical = inputs.physical; - let execution = inputs.selectable.executions()[index]; - let id = physical.physical_candidates[execution.candidate.0].logical_candidate_id; - physical - .logical_candidates - .memo - .candidate(id) - .input_requirements - .iter() - .find_map(|requirements| requirements.required_accuracy) -} - -/// `own` as an error per metric (by position in `AccuracyMetric::ALL`) -/// and a failure probability; 0 when `None`. -fn own_error_of(own: Option) -> ([f64; METRIC_COUNT], f64) { - let mut error_bound = [0.0; METRIC_COUNT]; - let Some(own) = own else { - return (error_bound, 0.0); - }; - let m = AccuracyMetric::ALL - .iter() - .position(|&listed| listed == own.metric) - .expect("AccuracyMetric::ALL lists every metric"); - error_bound[m] = own.max_error; - (error_bound, own.failure_probability) -} - /// What (K) is built over, per execution (by position in /// `SelectableExecutions::executions`), before any variable: its /// selectable input edges and upper bounds on the error and failure @@ -264,39 +431,38 @@ fn own_error_of(own: Option) -> ([f64; METRIC_COUNT], f64) { /// equivalent context, [`SharedVariables`]) and that a query with an /// accuracy target reaches has rows: a copy's rows would repeat its /// original's on the same variables. An execution whose inputs carry no -/// error under any selection (a readout, whose state is exact) has a fixed -/// error, its own, and no variable: its consumers read it as a constant -/// times $y$ ([`Composed`]). +/// error under any selection (a readout, whose state is exact) has a +/// constant error, its own, and no variable: its consumers read it as a +/// constant times $y$ ([`ErrorTerm`]). /// /// Example: B = p99 / p50 at relative 0.03, both read from DDSketch /// readouts required at relative 0.02 or 0.01: each readout's own error is /// its requirement, the ratio adds none and passes relative error on with /// factor 1/(1 − 0.1) per input, so its ceiling is 1.111 (0.02 + 0.02). pub(super) struct ErrorModel<'a> { - /// The rules (K) applies; `None` when no query has a target. - rules: Option<&'a AccuracyRules>, + /// The rules (K) applies. + rules: &'a AccuracyRules, /// Whether (K) has rows for the execution. has_rows: Vec, /// Per input, the selectable edges into the execution that have a /// $y$: (position in `SelectableExecutions::edges`, producer /// execution); empty for an execution without rows. - inputs: Vec>>, - /// $M$ per metric: the largest $E_{c,m}$ any selection gives. - error_bound_ceiling: Vec<[f64; METRIC_COUNT]>, - /// The largest $D_c$ any selection gives. - failure_probability_ceiling: Vec, - /// Whether the execution's error is fixed: its inputs carry none under - /// any selection, so its ceilings are its own error. - fixed: Vec, + selectable_edges_per_input: Vec>>, + /// $M$: the largest error and failure probability any selection gives + /// the execution. + ceiling: Vec, + /// Whether the execution's error is constant: its inputs carry none + /// under any selection, so its ceilings are its own error. + error_is_constant: Vec, } /// An execution's error in one metric, or its failure probability, as -/// (K) reads it: fixed (a readout's own requirement) or a variable with -/// its ceiling. +/// (K)'s rows read it: a constant (a readout's +/// own requirement) or a variable with its ceiling. #[derive(Clone, Copy)] -enum Composed { - Fixed(f64), - Variable(Variable, f64), +enum ErrorTerm { + Constant(f64), + Variable { variable: Variable, ceiling: f64 }, } impl<'a> ErrorModel<'a> { @@ -304,7 +470,7 @@ impl<'a> ErrorModel<'a> { /// kept and `shared` the copies of equivalent contexts. pub(super) fn new( inputs: &SelectionInputs<'_>, - rules: Option<&'a AccuracyRules>, + rules: &'a AccuracyRules, y: &BTreeMap, shared: &SharedVariables, ) -> Self { @@ -312,11 +478,11 @@ impl<'a> ErrorModel<'a> { let count = selectable.executions().len(); let has_rows: Vec = (0..count) .map(|index| { - rules.is_some_and(|rules| rules.targeted[index]) + rules.reached_by_query_with_target[index] && shared.original_of_execution[index] == index }) .collect(); - let inputs_of = |index: usize| -> Vec> { + let selectable_edges_of = |index: usize| -> Vec> { if !has_rows[index] { return Vec::new(); } @@ -333,107 +499,97 @@ impl<'a> ErrorModel<'a> { }; let mut model = Self { rules, - inputs: (0..count).map(inputs_of).collect(), + selectable_edges_per_input: (0..count).map(selectable_edges_of).collect(), has_rows, - error_bound_ceiling: vec![[0.0; METRIC_COUNT]; count], - failure_probability_ceiling: vec![0.0; count], - fixed: vec![false; count], + ceiling: vec![ExecutionError::default(); count], + error_is_constant: vec![false; count], }; - model.set_ceilings(inputs); - model.fixed = (0..count) + model.set_ceilings(); + model.error_is_constant = (0..count) .map(|c| { model.has_rows[c] - && model.inputs[c].iter().flatten().all(|&(_, u)| { - model.failure_probability_ceiling[u] == 0.0 - && model.error_bound_ceiling[u].iter().all(|&e| e == 0.0) - }) + && model.selectable_edges_per_input[c] + .iter() + .flatten() + .all(|&(_, u)| model.ceiling[u].is_zero()) }) .collect(); model } - /// Execution `u`'s error in metric `m` (by position in - /// `AccuracyMetric::ALL`); `None` when it is 0 under every selection. - fn error_bound(&self, accuracy: &AccuracyVariables, u: usize, m: usize) -> Option { - let ceiling = self.error_bound_ceiling[u][m]; - match (ceiling > 0.0, self.fixed[u]) { + /// Execution `u`'s error in metric `m` (by [`AccuracyMetric::index`]), + /// `None` when it is 0 under every selection. + fn error_bound(&self, accuracy: &AccuracyVariables, u: usize, m: usize) -> Option { + let ceiling = self.ceiling[u].error_bound_per_metric[m]; + match (ceiling > 0.0, self.error_is_constant[u]) { (false, _) => None, - (true, true) => Some(Composed::Fixed(ceiling)), - (true, false) => Some(Composed::Variable(accuracy.error_bound[&(u, m)], ceiling)), + (true, true) => Some(ErrorTerm::Constant(ceiling)), + (true, false) => Some(ErrorTerm::Variable { + variable: accuracy.error_bound[&(u, m)], + ceiling, + }), } } - /// Execution `u`'s failure probability; `None` when it is 0 under - /// every selection. - fn failure_probability(&self, accuracy: &AccuracyVariables, u: usize) -> Option { - let ceiling = self.failure_probability_ceiling[u]; - match (ceiling > 0.0, self.fixed[u]) { + /// Execution `u`'s failure probability; `None` when it is 0 + /// under every selection. + fn failure_probability(&self, accuracy: &AccuracyVariables, u: usize) -> Option { + let ceiling = self.ceiling[u].failure_probability; + match (ceiling > 0.0, self.error_is_constant[u]) { (false, _) => None, - (true, true) => Some(Composed::Fixed(ceiling)), - (true, false) => Some(Composed::Variable( - accuracy.failure_probability[&u], + (true, true) => Some(ErrorTerm::Constant(ceiling)), + (true, false) => Some(ErrorTerm::Variable { + variable: accuracy.failure_probability[&u], ceiling, - )), + }), } } - /// The ceilings, by rounds of the composition rule of (K) over the - /// selectable edges until none grows: at most one round per execution, - /// the longest path a selection without cycles (constraint (E)) has. - fn set_ceilings(&mut self, inputs: &SelectionInputs<'_>) { - let Some(rules) = self.rules else { - return; - }; - let with_rows: Vec = (0..self.has_rows.len()) - .filter(|&c| self.has_rows[c]) - .collect(); - let own: HashMap = with_rows - .iter() - .map(|&c| (c, own_error_of(own_error(inputs, c)))) - .collect(); - for _ in 0..=with_rows.len() { - let mut grew = false; - for &c in &with_rows { - let (mut error_bound, mut failure_probability) = own[&c]; - for (i, edges) in self.inputs[c].iter().enumerate() { - for (m, bound) in error_bound.iter_mut().enumerate() { - if let Some(contract) = rules.contract(c, m) { - let largest = edges - .iter() - .map(|&(_, u)| self.error_bound_ceiling[u][m]) - .fold(0.0, f64::max); - *bound += contract.error_factor_per_input[i] * largest; - } - } - failure_probability += edges - .iter() - .map(|&(_, u)| self.failure_probability_ceiling[u]) - .fold(0.0, f64::max); - } - for (ceiling, bound) in self.error_bound_ceiling[c].iter_mut().zip(error_bound) { - if bound > *ceiling { - *ceiling = bound; - grew = true; - } - } - if failure_probability > self.failure_probability_ceiling[c] { - self.failure_probability_ceiling[c] = failure_probability; - grew = true; - } - } - if !grew { - break; + /// The ceilings, in one pass over the executions with rows, producers + /// first ([`AccuracyRules::executions_producers_first`]): each + /// execution composed ([`AccuracyRules::compose`]) from the largest + /// ceiling of the producers selectable on each input. Every selectable + /// edge reads a group the memo orders before the consumer's (the memo + /// is acyclic, so no group reads itself), so each producer's ceiling is + /// final when its consumer reads it. + fn set_ceilings(&mut self) { + let rules = self.rules; + for &c in &rules.executions_producers_first { + if !self.has_rows[c] { + continue; } + let largest_per_input: Vec<(usize, ExecutionError)> = self.selectable_edges_per_input + [c] + .iter() + .enumerate() + .map(|(i, edges)| { + let largest = + edges + .iter() + .fold(ExecutionError::default(), |largest, &(_, u)| { + debug_assert_ne!( + rules.candidate_of_execution[u].group, + rules.candidate_of_execution[c].group, + "a selectable edge reads another group" + ); + largest.largest(self.ceiling[u]) + }); + (i, largest) + }) + .collect(); + // A ceiling bounds every selection, admitted or not. + self.ceiling[c] = rules.compose(c, largest_per_input).error; } } } -/// The accuracy variables of constraint (K), all continuous, each only for -/// an execution with rows ([`ErrorModel`]) whose error is not fixed and -/// whose ceiling is above 0, by execution position. +/// The accuracy variables of constraint (K), all continuous, in +/// each only for an execution with rows +/// ([`ErrorModel`]) whose error is not constant and whose ceiling is above +/// 0, by execution position. pub(super) struct AccuracyVariables { /// $E_{\kappa,c,m}$: the execution's error bound in metric $m$ (by - /// position in `AccuracyMetric::ALL`). + /// [`AccuracyMetric::index`]). error_bound: HashMap<(usize, usize), Variable>, /// $D_{\kappa,c}$: its failure probability. failure_probability: HashMap, @@ -446,15 +602,17 @@ impl AccuracyVariables { error_bound: HashMap::new(), failure_probability: HashMap::new(), }; - for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.fixed[c]) { - for m in 0..METRIC_COUNT { - if model.error_bound_ceiling[c][m] > 0.0 { + for c in + (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) + { + for (m, &ceiling) in model.ceiling[c].error_bound_per_metric.iter().enumerate() { + if ceiling > 0.0 { variables .error_bound .insert((c, m), problem.add(variable().min(0))); } } - if model.failure_probability_ceiling[c] > 0.0 { + if model.ceiling[c].failure_probability > 0.0 { variables .failure_probability .insert(c, problem.add(variable().min(0))); @@ -468,16 +626,10 @@ impl AccuracyVariables { pub(super) fn variable_count(&self) -> usize { self.error_bound.len() + self.failure_probability.len() } - - /// Whether (K) has no variable: no query with an accuracy target reads - /// an error. - pub(super) fn is_empty(&self) -> bool { - self.error_bound.is_empty() && self.failure_probability.is_empty() - } } -/// (K) (section 4, "Query composition and root checks"), per execution -/// $c$ with rows ([`ErrorModel`]): +/// (K) (`accuracy-models.md`, "Checking each query's target in +/// selection"), per execution $c$ with rows ([`ErrorModel`]): /// - $E_{c,m} \ge \varepsilon_c[m = m_c] + \sum_i \beta_{c,i,m} /// F_{c,i,m}$ and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per /// selectable edge, $F_{c,i,m} \le b_{c,i}$ where the contract's linear @@ -492,16 +644,19 @@ impl AccuracyVariables { /// \text{allowed}] + M(1 - z_{q,v})$, the metrics $g$ allows at the root /// ([`query_roots_with_target`]: the target's at an operator root, none /// for the operands of a scalar expression no contract covers), and -/// $D_v \le \delta_q + M(1 - z_{q,v})$ with $\delta_q$ per value -/// (`AccuracyTarget::failure_probability_per_value`); no $D$ row without -/// a per-value $\delta_q$. Each limit is widened by [`tolerated_limit`], -/// as the edge check widens it. +/// $D_v \le \delta_q + M(1 - z_{q,v})$ with $\delta_q$ per output value +/// (`AccuracyTarget::failure_probability_per_output_value`); no $D$ row +/// without one ([`RootLimits`], each limit widened by +/// [`tolerated_limit`]). +/// +/// It is [`AccuracyRules::compose`]'s rule over variables: the same own +/// error, factors, metric rule and validity caps. /// /// $F$ and $H$ are added to `problem` here. A copy of an equivalent context /// and its queries have no rows: they share their original's variables, /// and the copy's rows would be its original's /// ([`super::equivalent_contexts::copies_in_equivalent_contexts`] pairs -/// queries only with equal targets, $n_q$ and metrics allowed per output). +/// queries only with equal [`RootLimits`] per output). pub(super) fn accuracy_constraints( inputs: &SelectionInputs<'_>, model: &ErrorModel<'_>, @@ -511,17 +666,16 @@ pub(super) fn accuracy_constraints( shared: &SharedVariables, problem: &mut ProblemVariables, ) -> Vec { - let Some(rules) = model.rules else { - return Vec::new(); - }; + let rules = model.rules; let mut constraints = Vec::new(); - for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.fixed[c]) { + for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) + { // Σ_i β_{c,i,m} F_{c,i,m} per metric, and Σ_i H_{c,i}. - let mut error_of_inputs: [Expression; METRIC_COUNT] = Default::default(); + let mut error_of_inputs: [Expression; AccuracyMetric::COUNT] = Default::default(); let mut failure_probability_of_inputs = Expression::default(); - for (i, edges) in model.inputs[c].iter().enumerate() { + for (i, edges) in model.selectable_edges_per_input[c].iter().enumerate() { for (m, error_of_input) in error_of_inputs.iter_mut().enumerate() { - let read: Vec<(Variable, Composed)> = edges + let read: Vec<(Variable, ErrorTerm)> = edges .iter() .filter_map(|&(position, u)| { Some((y[&position], model.error_bound(accuracy, u, m)?)) @@ -534,8 +688,10 @@ pub(super) fn accuracy_constraints( // The metric rule: no error in m reaches c. for (y, error) in read { constraints.push(match error { - Composed::Fixed(_) => constraint!(y <= 0), - Composed::Variable(e, ceiling) => constraint!(e <= ceiling * (1 - y)), + ErrorTerm::Constant(_) => constraint!(y <= 0), + ErrorTerm::Variable { variable, ceiling } => { + constraint!(variable <= ceiling * (1 - y)) + } }); } continue; @@ -550,7 +706,16 @@ pub(super) fn accuracy_constraints( } *error_of_input += contract.error_factor_per_input[i] * of_input; } - let read: Vec<(Variable, Composed)> = edges + // H_{c,i} counts the failure of the one producer selected on + // input i, not of each readout value an output value depends + // on: `max(q)` over 1,000 series reads 1,000 readout values, + // and D here adds the readout's δ once. That is sound only + // because Pass 2's levels already split the query's δ over + // the readout values its output depends on + // (`readout_caps::query_levels`), so each readout's δ is the + // query's over 1,000; the root row checks the composed sum + // against the per-output-value δ. + let read: Vec<(Variable, ErrorTerm)> = edges .iter() .filter_map(|&(position, u)| { Some((y[&position], model.failure_probability(accuracy, u)?)) @@ -560,16 +725,17 @@ pub(super) fn accuracy_constraints( failure_probability_of_inputs += chosen_producers(&read, &mut constraints, problem); } } - let (own_error_bound, own_failure_probability) = own_error_of(own_error(inputs, c)); + let own = rules.own_error_per_execution[c]; for (m, of_inputs) in error_of_inputs.into_iter().enumerate() { if let Some(&error_bound) = accuracy.error_bound.get(&(c, m)) { - let own = own_error_bound[m]; + let own = own.error_bound_per_metric[m]; constraints.push(constraint!(error_bound >= own + of_inputs)); } } if let Some(&failure_probability) = accuracy.failure_probability.get(&c) { + let own = own.failure_probability; constraints.push(constraint!( - failure_probability >= own_failure_probability + failure_probability_of_inputs + failure_probability >= own + failure_probability_of_inputs )); } } @@ -583,16 +749,12 @@ pub(super) fn accuracy_constraints( if shared.original_of_query[query] != *query_id { continue; } - let Some(limits) = rules.roots.get(&(query, group)) else { + let Some(limits) = rules.root_limits(*query_id, group) else { continue; }; for &v in executions { let z = z[&(*query_id, v)]; - for (m, metric) in AccuracyMetric::ALL.iter().enumerate() { - let limit = match limits.allowed_metrics.contains(metric) { - true => tolerated_limit(limits.max_error), - false => 0.0, - }; + for (m, &limit) in limits.tolerated_error_per_metric.iter().enumerate() { constraints.extend( model .error_bound(accuracy, v, m) @@ -600,14 +762,10 @@ pub(super) fn accuracy_constraints( ); } if let (Some(limit), Some(failure_probability)) = ( - limits.failure_probability_per_value, + limits.tolerated_failure_probability_per_output_value, model.failure_probability(accuracy, v), ) { - constraints.extend(at_most_when_answering( - failure_probability, - tolerated_limit(limit), - z, - )); + constraints.extend(at_most_when_answering(failure_probability, limit, z)); } } } @@ -616,44 +774,44 @@ pub(super) fn accuracy_constraints( /// The error (or failure probability) one input of a consumer reads from /// the producer selected on it, `read` listing each producer's with its -/// $y$: $\sum_u e_u y_u$ over the producers with a fixed error (at most one -/// $y$ of an input is 1, constraint (A)), and over the others a variable -/// $F \ge E_u - M_u(1 - y_u)$, also at least that sum. +/// $y$: $\sum_u e_u y_u$ over the producers with a constant error (at most +/// one $y$ of an input is 1, constraint (A)), and over the others a +/// variable $F \ge E_u - M_u(1 - y_u)$, also at least that sum. fn chosen_producers( - read: &[(Variable, Composed)], + read: &[(Variable, ErrorTerm)], constraints: &mut Vec, problem: &mut ProblemVariables, ) -> Expression { - let fixed: Expression = read + let constant: Expression = read .iter() - .filter_map(|&(y, composed)| match composed { - Composed::Fixed(value) => Some(value * y), - Composed::Variable(..) => None, + .filter_map(|&(y, term)| match term { + ErrorTerm::Constant(value) => Some(value * y), + ErrorTerm::Variable { .. } => None, }) .sum(); if read .iter() - .all(|(_, composed)| matches!(composed, Composed::Fixed(_))) + .all(|(_, term)| matches!(term, ErrorTerm::Constant(_))) { - return fixed; + return constant; } let of_input = problem.add(variable().min(0)); - constraints.push(constraint!(of_input >= fixed)); - for &(y, composed) in read { - if let Composed::Variable(value, ceiling) = composed { - constraints.push(constraint!(of_input >= value - ceiling * (1 - y))); + constraints.push(constraint!(of_input >= constant)); + for &(y, term) in read { + if let ErrorTerm::Variable { variable, ceiling } = term { + constraints.push(constraint!(of_input >= variable - ceiling * (1 - y))); } } of_input.into() } -/// The root row bounding `composed` by `limit` when $z = 1$: none when the -/// ceiling is within it; $z = 0$ for a fixed value above it. -fn at_most_when_answering(composed: Composed, limit: f64, z: Variable) -> Option { - match composed { - Composed::Fixed(value) => (value > limit).then(|| constraint!(z <= 0)), - Composed::Variable(value, ceiling) => { - (limit < ceiling).then(|| constraint!(value <= limit + (ceiling - limit) * (1 - z))) +/// The root row bounding `term` by `limit` when $z = 1$: none when the +/// ceiling is within it; $z = 0$ for a constant above it. +fn at_most_when_answering(term: ErrorTerm, limit: f64, z: Variable) -> Option { + match term { + ErrorTerm::Constant(value) => (value > limit).then(|| constraint!(z <= 0)), + ErrorTerm::Variable { variable, ceiling } => { + (limit < ceiling).then(|| constraint!(variable <= limit + (ceiling - limit) * (1 - z))) } } } diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs index 784b6cfba..0a5113489 100644 --- a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -3,12 +3,12 @@ //! ([`copies_in_equivalent_contexts`]), so HiGHS does not branch over each //! copy. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; -use asap_accuracy::query_roots_with_target; use asap_types::accuracy::AccuracyTarget; use asap_types::stages::{ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId}; +use super::accuracy::{AccuracyRules, RootLimits}; use super::model::SelectionInputs; use super::COST_RELATIVE_TOLERANCE; use crate::selectable::QueryId; @@ -74,14 +74,13 @@ struct QuerySignature<'a> { /// Its accuracy target, by index among the workload's distinct targets /// ([`AccuracyTarget`] holds floats: metric, ε, δ and scope all equal). accuracy_target_index: usize, - /// $n_q$ (`MemoQuery::number_of_values_per_query_evaluation`), which - /// splits its target's failure probability per value in (K). - number_of_values_per_query_evaluation: Option, - /// Per output, whether the group it reads allows the target's metrics - /// at the root (`asap_accuracy::query_roots_with_target`: an operator - /// root does, an operand of `scalar(...) * 2` does not), which (K)'s - /// root rows read. - allows_target_metrics_per_output: Vec, + /// Per output, what (K)'s root check bounds it by + /// ([`AccuracyRules::root_limits`]); `None` without a target. Equal + /// limits give equal root rows: two queries at δ = 0.01 per value + /// returning 10 and 100 values have equal limits, over a whole result + /// they do not; an operator root and an operand of `scalar(...) * 2` + /// allow different metrics. + root_limits_per_output: Vec>, } /// Which executions, edges and queries share variables @@ -92,7 +91,7 @@ struct QuerySignature<'a> { /// same memo edges from producers that correspond, in the context or the /// same execution outside every evaluation context, and queries that pair /// up with the same output executions, exact-only groups, span, accuracy -/// target, $n_q$ and metrics allowed at each output, so the accuracy +/// target and root limits of (K) at each output (`rules`), so the accuracy /// constraint (K) has the same rows in both) and /// every cost the MILP reads in one (of a kept execution, or of an edge /// between two kept ones) is the same multiple $\alpha > 0$ of the @@ -117,9 +116,10 @@ struct QuerySignature<'a> { /// Example: the same queries every 60 s and every 300 s (one user each): /// the 300 s context's executions cost 1/5 of the 60 s ones' and share /// their variables. -pub(crate) fn copies_in_equivalent_contexts( - inputs: &SelectionInputs<'_>, +pub(crate) fn copies_in_equivalent_contexts<'a>( + inputs: &SelectionInputs<'a>, kept: &[bool], + rules: Option<&'a AccuracyRules>, ) -> SharedVariables { let selectable = inputs.selectable; let costs = inputs.costs; @@ -142,24 +142,17 @@ pub(crate) fn copies_in_equivalent_contexts( } } let memo = &inputs.physical.logical_candidates.memo; - // The (query, output group) pairs that allow the target's metrics. - let allowing_target_metrics: HashSet<(usize, MemoGroupId)> = query_roots_with_target(memo) - .into_iter() - .filter(|(_, _, _, metrics)| !metrics.is_empty()) - .map(|(query, group, _, _)| (query, group)) - .collect(); - // Per query, each output's answering executions and whether it allows - // the target's metrics. - let mut outputs_of: BTreeMap> = BTreeMap::new(); + // Per query, each output's answering executions and its root limits. + let mut outputs_of: BTreeMap> = BTreeMap::new(); for (&(query, group), (query_id, answering)) in memo.output_groups().iter().zip(selectable.answering()) { debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); - let allows = allowing_target_metrics.contains(&(query, group)); + let limits = rules.and_then(|rules| rules.root_limits(*query_id, group)); outputs_of .entry(*query_id) .or_default() - .push((answering, allows)); + .push((answering, limits)); } // Each evaluation context's queries; a query without an output has // none. @@ -198,7 +191,7 @@ pub(crate) fn copies_in_equivalent_contexts( }) }) .collect(); - let signature = |context: ExecutionContextId, query: QueryId| -> Option> { + let signature = |context: ExecutionContextId, query: QueryId| -> Option> { // A query has an evaluation context exactly when it has an output reference. let outputs = outputs_of[&query] .iter() @@ -211,11 +204,9 @@ pub(crate) fn copies_in_equivalent_contexts( .collect::>>()?; Some(QuerySignature { answering_executions_per_output: outputs, - number_of_values_per_query_evaluation: memo.queries()[query.0] - .number_of_values_per_query_evaluation, - allows_target_metrics_per_output: outputs_of[&query] + root_limits_per_output: outputs_of[&query] .iter() - .map(|&(_, allows)| allows) + .map(|&(_, limits)| limits) .collect(), exact_only_groups: inputs .physical @@ -228,7 +219,7 @@ pub(crate) fn copies_in_equivalent_contexts( }; // `context`'s key, and its queries in the order of their signatures in // it. - let key_of = |context: ExecutionContextId| -> Option<(ContextKey<'_>, Vec)> { + let key_of = |context: ExecutionContextId| -> Option<(ContextKey<'a>, Vec)> { let executions = executions_of[&context] .iter() .map(|&index| { @@ -329,7 +320,7 @@ pub(crate) fn copies_in_equivalent_contexts( .all(|&(a, b)| (b - alpha * a).abs() <= tolerance) }; // Per key, the originals, each with its queries in signature order. - let mut originals_of_key: HashMap, Vec<(ExecutionContextId, Vec)>> = + let mut originals_of_key: HashMap, Vec<(ExecutionContextId, Vec)>> = HashMap::new(); for © in executions_of.keys() { let Some((key, copy_queries)) = key_of(copy) else { diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index e91cec076..073e7feaf 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -7,7 +7,7 @@ //! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_rows`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | //! | retention (section 5) | $h_v$ | (R) [`super::retention::retention_constraints`] | //! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | -//! | accuracy (section 4), per execution | $E_{\kappa,c,m}$, $D_{\kappa,c}$ ([`AccuracyVariables`]); $F_{\kappa,c,i,m}$, $H_{\kappa,c,i}$ | (K) [`accuracy_constraints`], over [`ErrorModel`] | +//! | accuracy (`accuracy-models.md`, constraint (K)), per execution | $E_{\kappa,c,m}$, $D_{\kappa,c}$ ([`AccuracyVariables`]); $F_{\kappa,c,i,m}$, $H_{\kappa,c,i}$ | (K) [`accuracy_constraints`], over [`ErrorModel`] | //! //! The strongly connected components (E) needs are in [`super::graph`]; the //! executions left out before scaling, those dearer than a feasible plan @@ -412,20 +412,26 @@ pub(crate) fn solve_selection( .map(|((&dearer, dominated), cost)| cost.is_ok() && !dearer && !dominated) .collect(); let components = Components::of(inputs); - let shared = copies_in_equivalent_contexts(inputs, &kept); + let shared = copies_in_equivalent_contexts(inputs, &kept, rules.as_ref()); let mut problem = ProblemVariables::new(); let variables = selection_variables(inputs, &kept, &shared, &components, &mut problem); - let error_model = ErrorModel::new(inputs, rules.as_ref(), &variables.y, &shared); - let accuracy = AccuracyVariables::new(&error_model, &mut problem); - let accuracy_rows = accuracy_constraints( - inputs, - &error_model, - &accuracy, - &variables.y, - &variables.z, - &shared, - &mut problem, - ); + // (K), only when some query has an accuracy target. + let accuracy_rows = rules + .as_ref() + .map(|rules| { + let error_model = ErrorModel::new(inputs, rules, &variables.y, &shared); + let accuracy = AccuracyVariables::new(&error_model, &mut problem); + accuracy_constraints( + inputs, + &error_model, + &accuracy, + &variables.y, + &variables.z, + &shared, + &mut problem, + ) + }) + .unwrap_or_default(); // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs @@ -514,12 +520,13 @@ pub(crate) fn solve_selection( model.set_option("mip_abs_gap", 0.0); model.set_option("output_flag", false); model.set_option("threads", 1); - if !accuracy.is_empty() { - // Errors and failure probabilities are small (1e-3, 1e-7): the - // default feasibility tolerance 1e-7 would let (K) pass a bound - // above its limit. - model.set_option("primal_feasibility_tolerance", 1e-10); - model.set_option("mip_feasibility_tolerance", 1e-10); + if let Some(tolerance) = rules + .as_ref() + .and_then(|rules| rules.feasibility_tolerance()) + { + // (K)'s limits can be far below HiGHS's absolute default 1e-7. + model.set_option("primal_feasibility_tolerance", tolerance); + model.set_option("mip_feasibility_tolerance", tolerance); } let construction_ms = started.elapsed().as_millis() as u64; let started = Instant::now(); diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 517877ee3..92ce6f79f 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeSet, HashMap}; -use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; @@ -776,7 +776,8 @@ fn contexts_whose_queries_read_different_groups_are_not_copies() { selectable: &selectable, costs: &costs, }; - let shared = copies_in_equivalent_contexts(&inputs, &vec![true; selectable.executions().len()]); + let shared = + copies_in_equivalent_contexts(&inputs, &vec![true; selectable.executions().len()], None); assert!(shared .original_of_execution .iter() @@ -1021,7 +1022,7 @@ fn a_context_equivalent_to_another_shares_its_variables() { selectable: &selectable, costs, }; - copies_in_equivalent_contexts(&inputs, &kept) + copies_in_equivalent_contexts(&inputs, &kept, None) }; let SharedVariables { original_of_execution, @@ -1071,6 +1072,7 @@ fn a_context_equivalent_to_another_shares_its_variables() { costs: &costs, }, &kept, + None, ); assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(1)]); assert!(shared @@ -1142,6 +1144,7 @@ fn contexts_whose_small_edge_costs_differ_by_rounding_are_copies() { costs: &costs, }, &vec![true; executions.len()], + None, ); assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(0)]); for (index, &original) in shared.original_of_execution.iter().enumerate() { @@ -1285,8 +1288,8 @@ fn relative(max_error: f64) -> AccuracyTarget { AccuracyTarget::max_error(max_error).accuracy_metric(AccuracyMetric::Relative) } -/// #1207's query B, p99 / p50 at relative 0.03 (concepts doc, section 4, -/// "Example: allocation in a ratio"), over readouts required at relative +/// Query B, p99 / p50 at relative 0.03 (`accuracy-models.md`, "Example: +/// allocation in a ratio"), over readouts required at relative /// 0.02 (cost 1) or 0.01 (cost 3) each: 1.111 (0.02 + 0.02) = 0.044 and /// 1.111 (0.02 + 0.01) = 0.033 are above 0.03, so (K) selects both 0.01 /// readouts (0.022). The greedy plan reads the two 0.02 readouts (cost 2) @@ -1343,7 +1346,8 @@ fn k_allocates_a_ratio_s_error_among_its_readouts() { // (K)'s E and D exist only in the 10 s context (the 20 s one is // its copy), and only for the ratio: a readout's error is fixed. let kept = vec![true; selectable.executions().len()]; - let shared = copies_in_equivalent_contexts(&inputs, &kept); + let rules = AccuracyRules::of(&inputs).unwrap(); + let shared = copies_in_equivalent_contexts(&inputs, &kept, Some(&rules)); let mut problem = ProblemVariables::new(); let variables = selection_variables( &inputs, @@ -1352,8 +1356,7 @@ fn k_allocates_a_ratio_s_error_among_its_readouts() { &Components::of(&inputs), &mut problem, ); - let rules = AccuracyRules::of(&inputs); - let model = ErrorModel::new(&inputs, rules.as_ref(), &variables.y, &shared); + let model = ErrorModel::new(&inputs, &rules, &variables.y, &shared); let accuracy = AccuracyVariables::new(&model, &mut problem); let one_context = 2; assert_eq!(accuracy.variable_count(), one_context, "{entries:?}"); @@ -1541,3 +1544,88 @@ fn contexts_whose_roots_allow_different_metrics_are_not_copies() { let (solved, _) = solve(&physical, &[every(10_000), every(20_000)], &w, &[], &[]); assert_eq!(solved.unwrap_err(), SelectionMilpError::Infeasible); } + +/// The same readout every 10 s and every 20 s, returning 10 values every +/// 10 s and 100 every 20 s, at relative 0.01 without a failure probability +/// or with δ = 0.01 per value: (K) reads the same root limits for both +/// (no per-value δ, or 0.01 per value whatever $n_q$), so the 20 s context +/// is a copy of the 10 s one. +#[test] +fn contexts_whose_queries_differ_only_in_values_not_split_by_k_are_copies() { + let per_value = relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + for target in [relative(0.01), per_value] { + let mut memo = empty_memo(); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1.0)]); + for values in [10, 100] { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some(target.clone()), + number_of_values_per_query_evaluation: Some(values), + }); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if let asap_types::stages::ExecutionContext::Evaluation { grid } = + selectable.context(selectable.executions()[index].context) + { + if grid.period_ms == 20_000 { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + } + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs); + let shared = copies_in_equivalent_contexts(&inputs, &kept, rules.as_ref()); + assert_eq!( + shared.original_of_query, + vec![QueryId(0), QueryId(0)], + "{target:?}" + ); + } +} + +/// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of 1e4 +/// values, 1e-6 per value, over readouts at relative 0.01 failing with +/// 5.4e-7 (cost 1) or 4e-7 (cost 3) each: two at 5.4e-7 fail together with +/// up to 1.08e-6, above 1e-6 by 8e-8, within HiGHS's default feasibility +/// tolerance 1e-7; one of each, 9.4e-7, is within it. (K) solves at 1e-9 +/// (`AccuracyRules::feasibility_tolerance`), so it selects one of each. +#[test] +fn k_bounds_a_per_value_failure_probability_below_the_default_solver_tolerance() { + let mut memo = empty_memo(); + let levels = [(0.01, 5.4e-7), (0.01, 4e-7)]; + let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); + let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(ratio), + accuracy_target: Some(relative(0.03).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(10_000), + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let mut w = vec![0.0; physical.physical_candidates.len()]; + for readouts in [&p99_readouts, &p50_readouts] { + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + } + let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); + let selected = candidates(&solved.unwrap()); + let count = |level: usize| { + [&p99_readouts, &p50_readouts] + .iter() + .filter(|readouts| selected.contains(&physical_of(&physical, readouts[level]))) + .count() + }; + assert_eq!((count(0), count(1)), (1, 1), "{selected:?}"); +} diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index ae7c9f997..9a8d8bf09 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -4,7 +4,6 @@ use std::collections::{BTreeSet, HashMap}; -use asap_accuracy::readout_error_metric; use asap_cost::node::analytical::AnalyticalCostError; use asap_cost::{summary_type_name, CostModel}; use asap_types::accuracy::AccuracyMetric; @@ -118,8 +117,11 @@ pub struct AcceptedAccuracy { /// Whether the query has an accuracy target. pub has_accuracy_target: bool, /// The metric of the accepted error bound, that of the first sketch - /// readout the output reads; `None` for an exact output, which meets - /// every metric, and for a query without an accuracy target. + /// readout the output reads (in plan order; with a target naming no + /// metric, readouts in several metrics may each be within it, as (K) + /// bounds each metric apart, and this is the first's); `None` for an + /// exact output, which meets every metric, and for a query without an + /// accuracy target. pub accepted_metric: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. @@ -234,8 +236,9 @@ pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { /// Per query, the sketches its output reads in `plan` (each sketch build /// below its root, in node order) and, when it has an accuracy target and /// reads a sketch, the metric of the first sketch readout below its root -/// (`asap_accuracy::readout_error_metric`), whatever metric the target -/// names: a target naming none reports the one it accepted. +/// (`CandidateForMemoGroup::required_accuracy`, the requirement (K) +/// checked), whatever metric the target names: a target naming none +/// reports the one it accepted. pub(crate) fn accepted_accuracy( physical: &PhysicalASAPMemoCandidates, plan: &PhysicalASAPDAG, @@ -274,22 +277,12 @@ pub(crate) fn accepted_accuracy( sketches.push(name); } } - Operator::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { - // The algorithm of the state group the readout's - // candidate reads. - let algorithm = node.selected_physical_candidate.and_then(|v| { + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) => { + // The metric of the error the readout's candidate + // was required to meet, the one (K) checked. + readout_metrics.extend(node.selected_physical_candidate.and_then(|v| { let id = physical.physical_candidates[v.0].logical_candidate_id; - match memo.candidate(id).expression.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, .. - }) => memo.groups()[summary_input.0 as usize] - .group_properties - .sketch_algorithm_of_state(), - _ => None, - } - }); - readout_metrics.extend(algorithm.and_then(|algorithm| { - readout_error_metric(&FieldDataType::Sketch(algorithm), query) + Some(memo.candidate(id).required_accuracy()?.metric) })); } _ => {} diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs index 48145973e..5ed6aa715 100644 --- a/crates/types/src/accuracy/target/mod.rs +++ b/crates/types/src/accuracy/target/mod.rs @@ -64,6 +64,18 @@ impl AccuracyMetric { Self::Relative, Self::Rank, ]; + /// How many metrics there are: `ALL`'s length, the size of an array + /// holding one value per metric. + pub const COUNT: usize = Self::ALL.len(); + + /// The position of `self` in [`Self::ALL`], the index of its entry in + /// an array of one value per metric: `Relative` is 3. + pub fn index(self) -> usize { + Self::ALL + .iter() + .position(|&listed| listed == self) + .expect("AccuracyMetric::ALL lists every metric") + } } /// The wire name of an [`AccuracyMetric`]. @@ -285,20 +297,24 @@ impl AccuracyTarget { self.max_error == 0.0 && self.failure_probability == Some(0.0) } - /// The target's failure probability per value: its δ split evenly over - /// the values its scope covers when one evaluation returns - /// `values_per_evaluation` ($n_q$; [`AccuracyScope::max_values_covered`]); - /// 1 when it sets no δ. `None` when that count is not known for a δ over - /// the whole result: no per-value δ follows (Pass 2 then gives the - /// query's readouts no level). + /// The failure probability each output value of one evaluation must + /// meet: the target's δ split evenly, by the union bound, over the + /// values its scope covers when one evaluation returns + /// `values_per_evaluation` ($n_q$; [`AccuracyScope::max_values_covered`]). + /// `None` when the target sets no δ, and when that count is not known + /// for a δ over the whole result. A readout value's failure probability + /// splits δ further over the readout values an output value depends + /// on, and is built on this one. /// /// Example: δ = 0.01 over the whole result of a query returning 100 - /// series gives 0.0001 per value; with the number of series unknown, - /// `None`; per value (`AccuracyScope::PerValue`), 0.01 either way. - pub fn failure_probability_per_value(&self, values_per_evaluation: Option) -> Option { - let Some(delta) = self.failure_probability else { - return Some(1.0); - }; + /// series gives 0.0001 per output value; with the number of series + /// unknown, `None`; per value (`AccuracyScope::PerValue`), 0.01 either + /// way. + pub fn failure_probability_per_output_value( + &self, + values_per_evaluation: Option, + ) -> Option { + let delta = self.failure_probability?; let values = self .accuracy_scope .max_values_covered(values_per_evaluation)?; diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs index 5c53a758a..290b10a52 100644 --- a/crates/types/src/accuracy/target/tests.rs +++ b/crates/types/src/accuracy/target/tests.rs @@ -334,16 +334,23 @@ fn exact_targets() { } /// A whole-result δ splits over the values one evaluation returns; a -/// per-value δ does not; no δ is 1; an unknown count has none. +/// per-value δ does not; no δ, or a whole-result one over an unknown count, +/// has none. #[test] -fn a_whole_result_failure_probability_splits_over_the_values() { +fn a_whole_result_failure_probability_splits_over_the_output_values() { let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); - assert_eq!(whole.failure_probability_per_value(Some(100)), Some(1e-4)); - assert_eq!(whole.failure_probability_per_value(None), None); + assert_eq!( + whole.failure_probability_per_output_value(Some(100)), + Some(1e-4) + ); + assert_eq!(whole.failure_probability_per_output_value(None), None); let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); - assert_eq!(per_value.failure_probability_per_value(None), Some(0.01)); assert_eq!( - AccuracyTarget::max_error(0.01).failure_probability_per_value(None), - Some(1.0) + per_value.failure_probability_per_output_value(None), + Some(0.01) + ); + assert_eq!( + AccuracyTarget::max_error(0.01).failure_probability_per_output_value(Some(100)), + None ); } diff --git a/crates/types/src/stages/logical/candidate/mod.rs b/crates/types/src/stages/logical/candidate/mod.rs index fae0c4d41..a9e646f9a 100644 --- a/crates/types/src/stages/logical/candidate/mod.rs +++ b/crates/types/src/stages/logical/candidate/mod.rs @@ -4,7 +4,7 @@ use crate::ir::operator::asap::summary_type_of_operator; use crate::ir::schema::FieldDataType; use crate::ir::Operator; -use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId}; +use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId, RequiredAccuracy}; use crate::summary_type::{parameter_list, SketchAlgorithm, SketchParams}; /// One candidate: position `index_in_group` in group `group`'s candidates @@ -58,6 +58,19 @@ impl CandidateForMemoGroup { origin, } } + + /// The accuracy a readout requires of the state it reads, which is + /// also the error the readout adds: its input's `required_accuracy`. + /// `None` for any other candidate, and for a readout no query with an + /// accuracy target gives a level. + /// + /// Example: a DDSketch quantile readout Pass 2 gives relative 0.01 at + /// δ = 1e-3 returns that requirement; a scan returns `None`. + pub fn required_accuracy(&self) -> Option { + self.input_requirements + .iter() + .find_map(|requirements| requirements.required_accuracy) + } } /// The θ of each build candidate of `build`: every listed θ diff --git a/crates/types/src/stages/logical/group/column_properties.rs b/crates/types/src/stages/logical/group/column_properties.rs index b3dd1dd6d..30115e8d2 100644 --- a/crates/types/src/stages/logical/group/column_properties.rs +++ b/crates/types/src/stages/logical/group/column_properties.rs @@ -50,6 +50,15 @@ pub enum ValueSign { Unknown, } +impl ValueSign { + /// Whether every value has one sign (`NonNegative` or `NonPositive`): + /// then a relative error ε on each value is at most ε on their sum, and + /// DDSketch's relative bound on a quantile holds. + pub fn is_one_sign(self) -> bool { + matches!(self, Self::NonNegative | Self::NonPositive) + } +} + /// Where an output column's values come from, a group property derived /// from the input groups (concepts doc, section 3, "Column lineage"). It /// names a column across groups where a schema field's name cannot: two diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 5b52d860c..3637934d5 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -22,7 +22,7 @@ use crate::stages::{ LogicalRequirements, MemoGroup, MemoGroupId, }; use crate::summary_type::SketchStatistic; -use crate::workload::bounds::source_name; +use crate::workload::bounds::{declared_max_distinct_per_evaluation, source_name}; use crate::workload::{series_facts, DataArrival, DataWorkload, SourceFacts}; use hash_consing::{ expression_hash, same_candidate, same_serialized_expression, serialized_expression, @@ -129,20 +129,20 @@ impl Memo { /// rows counts in one evaluation: the sum of the bounds the data /// workload declares for the sources scanned below it /// (`GroupProperties::scanned_sources`, - /// `SourceFacts::max_distinct_per_evaluation`); `None` when one of them + /// [`declared_max_distinct_per_evaluation`]); `None` when one of them /// declares none. /// /// Example: `count(distinct user_id)` over `Scan events` with `events` /// bounded at 9,636 gives 9,636 for every group above the scan. pub fn max_distinct_per_evaluation(&self, group: MemoGroupId) -> Option { - self.groups[group.0 as usize] - .group_properties - .scanned_sources - .iter() - .try_fold(0u64, |total, source| { - let (facts, _) = series_facts(&self.source_facts, source_name(source))?; - total.checked_add(facts.max_distinct_per_evaluation?) - }) + declared_max_distinct_per_evaluation( + &self.source_facts, + self.groups[group.0 as usize] + .group_properties + .scanned_sources + .iter() + .map(source_name), + ) } /// The groups the `by` keys of the state `group` is, or reads, form diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 32bff6d3b..d43517f9c 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -13,7 +13,9 @@ use crate::ir::{ use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; use crate::stages::ValueSign; use crate::summary_type::SketchStatistic; -use crate::workload::{DataWorkload, PrometheusMetricType, SourceFacts, ValueMagnitudeRange}; +use crate::workload::{ + series_facts, DataWorkload, PrometheusMetricType, SourceFacts, ValueMagnitudeRange, +}; /// A scan source's name: its PromQL metric or SQL table. pub fn source_name(source: &Source) -> &str { @@ -474,6 +476,26 @@ pub fn declared_value_magnitude_range<'a>( }) } +/// The most distinct items one distinct-count summary over the sources +/// `scanned` counts per evaluation: the sum of the bound each declares +/// ([`SourceFacts::max_distinct_per_evaluation`]), each looked up as a +/// PromQL selector reads it ([`DataWorkload::series_facts`]: a histogram +/// family `foo`'s bound covers `foo_bucket`, `foo_sum` and `foo_count`, as +/// its `series_count` does). Unlike [`declared_value_magnitude_range`], +/// whose ranges differ between a family's series, a bound on the distinct +/// items counted holds for each series of the family. `None` when one of +/// them declares none, or the sum overflows. E.g. `events` bounded at +/// 9,636 and `clicks` at 1,000 give 10,636. +pub(crate) fn declared_max_distinct_per_evaluation<'a>( + sources: &BTreeMap, + scanned: impl IntoIterator, +) -> Option { + scanned.into_iter().try_fold(0u64, |total, source| { + let (facts, _) = series_facts(sources, source)?; + total.checked_add(facts.max_distinct_per_evaluation?) + }) +} + /// Rows per millisecond the sources in `sources` receive together, each /// at its own rate: `series_count` series each sampled every /// `sample_interval`, or, for a source that does not declare both, its diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index c072a20c5..400599972 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -182,7 +182,7 @@ The built-in models currently include: |---|---| | Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require error propagation | | KLL | Normalized rank error from the empirical 99% calibration `2.296 / k^0.9723`, fitted on Apache DataSketches' C++ KLL (`a9b42755072b`); below a failure probability of 0.01 that error is scaled by the Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`, a model rather than a measurement, and the guarantee names that contract apart | -| DDSketch | Relative value error from alpha under the supported estimator/domain contract; a quantile's bound holds only over values of one sign (the state column's `column_signs` non-negative or non-positive), else the readout has no guarantee | +| DDSketch | Relative value error from alpha under the supported estimator/domain contract; a quantile's bound holds only over values of one sign (the state column's `column_signs` non-negative or non-positive, "Sign conditions"), else its failure probability is unknown and the readout serves no query with an accuracy target | | Classic HLL | Relative error certified at the required failure probability over a source declaring at most `max_distinct_per_evaluation` distinct items (summed over the sources scanned below the state), from the linear-counting collision bound under independent uniform bucket hashes; without such a bound, or above the contract's range (`m/2` items at precision `p`), only the RSE magnitude with unknown failure probability | | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | @@ -264,6 +264,30 @@ them, and sharing does not justify reducing the failure budget. Per-result guarantees also do not imply a simultaneous guarantee across every query or evaluation time in a dashboard. +### Sign conditions + +Two rules hold only over values of one sign. Both read the group property +`column_signs` (declared per source column in `SourceFacts::column_signs`, a +counter's value non-negative, and derived through each operator: a negation +flips it, `abs` makes it non-negative) and test it with +`ValueSign::is_one_sign`, non-negative or non-positive: + +- DDSketch's relative bound on a quantile. The state column's sign + (`GroupProperties::state_column_sign`, the sign of the values the states + were updated with) must be one sign; otherwise the guarantee + (`guarantee/ddsketch.rs`) keeps its metric, relative error, with an unknown + failure probability, so the readout serves no query with an accuracy target + (`RootPruneReason::NoGuarantee`) and `theoretical_guarantee` returns none. + `quantile_over_time(0.99, latency[1h])` with `latency` declared + non-negative holds at α; over a gauge of unknown sign it does not. +- A sum's or an average's relative error (the contracts table): the input + column's sign must be one sign, as `|Σ (x̂_i − x_i)| ≤ ε Σ |x_i| = + ε |Σ x_i|` holds then, while values of unknown sign may cancel. + +The cost model reads whether a DDSketch state's values are non-negative from +the sources' declared signs instead, so it can disagree with this rule where +an operator changes the sign (DDSketch over `-latency`). + ## Readout requirements for composed queries Pass 2 gives each readout the accuracy it is required to meet. A query's @@ -340,6 +364,98 @@ An exact target gets no approximate level. A model with only a fixed 1% failure contract cannot meet a level that needs a smaller failure probability; another supported configuration or candidate is needed. +## Checking each query's target in selection: constraint (K) + +Pass 2 gives each readout requirement levels, and the logical-edge check keeps +the producers that meet them. A composed query reads several readouts, each +at one of its levels, and only the selection decides which: Stage 3's +selection MILP checks that the readouts a selection combines compose within +each query's target. This is constraint (K). + +**Variables.** Per execution `c` (a candidate in one evaluation context) +that a query with a target reaches, and that is its own original (not in a +copy of an equivalent context): its error bound `E_{c,m}` per metric `m` and +its failure probability `D_c`, continuous; per input `i`, `F_{c,i,m}` and +`H_{c,i}`, what it reads from the producer selected on `i`. + +**Rows.** With `y_{u,c,i}` the edge variables and `z_{q,v}` the choice of +execution `v` answering query `q`: + +- `E_{c,m} ≥ ε_c[m = m_c] + Σ_i β_{c,i,m} F_{c,i,m}`, with + `F_{c,i,m} ≥ E_{u,m} − M_{u,m}(1 − y_{u,c,i})` per selectable producer `u`; + `ε_c` is a readout's own error, its required accuracy + (`CandidateForMemoGroup::required_accuracy`), 0 for any other candidate. +- The metric rule on the selected path: `E_{u,m} ≤ M_{u,m}(1 − y_{u,c,i})` + when `c` has no contract in `m`. An input with `β = 0` passes none. +- The validity cap: `F_{c,i,m} ≤ b_{c,i}` where the contract's linear bound + holds only up to `b` (a ratio's denominator). +- `D_c ≥ δ_c + Σ_i H_{c,i}`, `H_{c,i} ≥ D_u − M_u(1 − y_{u,c,i})`: a union + bound over the producers selected. +- At each output of a query with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` + in each metric the root allows (the target's at an operator root, none for + the operands of a scalar expression no contract covers, 0 elsewhere), and + `D_v ≤ δ_q / V + M(1 − z_{q,v})` with `V` the values the target's scope + covers (`AccuracyTarget::failure_probability_per_output_value`, `n_q` for a + whole result); no `D` row without a δ. Each limit is widened as the edge + check widens it (`tolerated_limit`). + +`D` adds each selected producer's δ once, not once per readout value an +output value depends on: `max(q)` over 1,000 series reads 1,000 readout +values. That is sound because Pass 2's levels already split the query's δ +over the readout values its output can depend on, so each readout's δ is the +query's over 1,000. + +**Constant errors.** A readout's state is exact, so its error is its own +requirement under every selection: it has no `E` or `D` variable, and its +consumers read `Σ_u ε_u y_{u,c,i}` (at most one `y` of an input is 1). This is +exactly the variable form with `E_u = ε_u`, and it removes the big-M rows of +every readout. + +**Ceilings.** `M` is the largest error any selection gives an execution, +computed in one pass in the memo's topological order (every selectable edge +reads a group ordered before the consumer's), each input taking its largest +producer's ceiling, by the same composition rule as the rows. The rule is +written once (`AccuracyRules::compose`), and also checks a whole selection +(`AccuracyRules::selection_meets_targets`): the greedy upper bound that prunes +executions dearer than a feasible plan uses its plan only when it meets (K), +since the cheapest readouts of a ratio may each meet their level and compose +above the target. + +**Solver tolerance.** HiGHS's feasibility tolerances are absolute, 1e-7 by +default, while a per-value δ is often far smaller: δ = 0.01 over a whole +result of 1e4 values is 1e-6 per value. With a target, HiGHS's primal and MIP +feasibility tolerances are `min(1e-7, 1e-3 · ℓ)` for the smallest positive +limit `ℓ` a target sets, so a row may pass its limit by at most a thousandth +of the smallest one; where that is not below 1e-7, HiGHS keeps its defaults. +HiGHS accepts no tolerance below 1e-10, so a per-value δ below 1e-7 is +checked only to 1e-10. Scaling (K)'s rows by their limits instead kept the defaults but +solved RangeHeavy-8users about 4× slower. + +**Context copies.** A copy of an equivalent evaluation context shares its +original's variables and has no rows of its own. Two contexts are copies only +when each output's root limits (the tolerated error per metric and failure +probability per output value) are equal, so the copy's rows would repeat the +original's: queries at δ = 0.01 per value returning 10 and 100 values are +copies; over a whole result they are not. + +### Example: allocation in a ratio + +A is `quantile_over_time(0.95, latency[1h])` at rank error 0.01, and B is +`quantile_over_time(0.99, latency[1h]) / quantile_over_time(0.5, latency[1h])` +at relative error 0.03, over non-negative `latency`. A reads a KLL readout at +rank 0.01. B's ratio passes each operand's relative error on with factor +`1/(1 − 0.1) = 1.111`, so Pass 2 gives each DDSketch readout below it the +levels of the grid up to its cap; with levels 0.02 (cost 1) and 0.01 (cost 3): + +| p99 readout | p50 readout | B's composed error | meets 0.03 | +|---|---|---|---| +| 0.02 | 0.02 | 1.111 · 0.04 = 0.044 | no | +| 0.02 | 0.01 | 1.111 · 0.03 = 0.033 | no | +| 0.01 | 0.01 | 1.111 · 0.02 = 0.022 | yes | + +(K) selects both 0.01 readouts, cost 6. A greedy plan reading the two +cheapest readouts (cost 2) misses B's target, so it bounds nothing. + ## Evidence and trust boundaries `AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the @@ -448,15 +564,14 @@ accuracy/src/ │ # the error contracts, each readout's required-accuracy │ # levels; MAX_ERROR_LEVELS ├── logical_edge/ # ErrorMetricsOfMemo::edge_meets_logical_requirements -│ # (α^log), with DDSketch's sign condition; -│ # within_limit and tolerated_limit, the requirement -│ # comparison +│ # (α^log); within_limit and tolerated_limit, the +│ # requirement comparison └── guarantee/ ├── mod.rs # theoretical_guarantee (one lookup at a failure │ # probability per algorithm) ├── binomial.rs # binomial_upper_tail ├── kll/ - ├── ddsketch.rs + ├── ddsketch.rs # relative error α; its sign condition ├── hll/ # HLL: classic certificate over a distinct bound, else │ # RSE (unknown failure probability) ├── cms/ From 593ef4163136ce897265a95d6f464cba5ade8865 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:45:02 +0000 Subject: [PATCH 3/4] fix(stage3): solve (K) at tolerance 1e-10 and check the selection exactly; a target below the solver's precision is an error (#1207 PR 8, review round 2) --- crates/stage3-plan-selection/src/lib.rs | 7 + .../src/milp/accuracy.rs | 144 +++++++----------- .../src/milp/model/mod.rs | 50 +++++- .../src/milp/model/tests.rs | 63 ++++++-- .../src/milp/upper_bound.rs | 8 +- crates/stage3-plan-selection/src/report.rs | 12 ++ crates/types/src/stages/pruning.rs | 5 +- docs/design_docs/concepts/accuracy-models.md | 19 ++- 8 files changed, 188 insertions(+), 120 deletions(-) diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index e00aa207c..c50c77014 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -119,6 +119,13 @@ pub fn select_plan( .map_err(|error| match error { SelectionMilpError::Infeasible => infeasible(&costs), SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), + SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value, + } => ErrorSelectingPlan::AccuracyTargetBelowSolverPrecision { + query: query.0, + failure_probability_per_output_value, + }, })?; let plan = plan_of(physical, &solved, &outputs).map_err(|error| { ErrorSelectingPlan::Solver(format!("the selected plan is invalid: {error}")) diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index 80827aa9e..5f530e122 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -4,7 +4,8 @@ //! per error metric, over the error each execution adds and passes on //! ([`AccuracyRules`], [`ErrorModel`]), and the same rule checked on one //! selection ([`AccuracyRules::selection_meets_targets`]), which the greedy -//! upper bound needs to be a feasible plan of (K). One composition rule, +//! upper bound needs to be a feasible plan of (K) and which checks the +//! solver's selection exactly. One composition rule, //! [`AccuracyRules::compose`], gives both that check and the ceilings of //! (K)'s big-M rows; [`accuracy_constraints`] writes the same rule as rows. @@ -91,11 +92,6 @@ pub(super) struct AccuracyRules { /// Per (query, output group its root reads), what its root check /// bounds the output by. roots: HashMap<(QueryId, MemoGroupId), RootLimits>, - /// The smallest positive limit a target sets (an ε, a δ per output - /// value), before [`tolerated_limit`] widens it; `None` when none does - /// (exact targets only). The solver's feasibility tolerance is set - /// from it ([`Self::feasibility_tolerance`]). - smallest_positive_limit: Option, } /// What a root check of (K) bounds one query output by, each limit @@ -209,13 +205,6 @@ impl AccuracyRules { (0..candidate_of_execution.len()).collect(); executions_producers_first .sort_by_key(|&c| position_in_order[candidate_of_execution[c].group.0 as usize]); - let mut smallest_positive_limit: Option = None; - let mut see_limit = |limit: f64| { - if limit > 0.0 { - smallest_positive_limit = - Some(smallest_positive_limit.map_or(limit, |smallest| smallest.min(limit))); - } - }; let roots = query_roots_with_target(memo) .into_iter() .map(|(query, root, target, allowed_metrics)| { @@ -225,10 +214,6 @@ impl AccuracyRules { let mut tolerated_error_per_metric = [0.0; AccuracyMetric::COUNT]; for metric in allowed_metrics { tolerated_error_per_metric[metric.index()] = tolerated_limit(target.max_error); - see_limit(target.max_error); - } - if let Some(limit) = failure_probability { - see_limit(limit); } let limits = RootLimits { tolerated_error_per_metric, @@ -245,28 +230,9 @@ impl AccuracyRules { reached_by_query_with_target, executions_producers_first, roots, - smallest_positive_limit, }) } - /// The primal and MIP feasibility tolerance HiGHS solves (K) at: - /// $\min(10^{-7}, 10^{-3} \cdot \ell)$ for the smallest positive limit - /// $\ell$ a target sets, so a row may pass its limit by at most a - /// thousandth of the smallest one, but not below 1e-10, the least - /// HiGHS accepts (a per-value δ under 1e-7 is then checked only to - /// 1e-10); `None` (HiGHS's defaults) when it is not below 1e-7. - /// HiGHS's tolerances are absolute: δ = 0.01 over a whole result of 1e4 - /// values is 1e-6 per value, of which the default 1e-7 is a tenth. - /// Scaling (K)'s rows by their limits instead kept the defaults but - /// solved RangeHeavy-8users about 4× slower (larger big-M - /// coefficients). - /// - /// Example: relative 0.05 with δ = 0.01 over 1e4 values: 1e-9. - pub(super) fn feasibility_tolerance(&self) -> Option { - let tolerance = (1e-3 * self.smallest_positive_limit?).max(1e-10); - (tolerance < 1e-7).then_some(tolerance) - } - /// What the root check of `query`'s output reading `group` bounds it /// by; `None` when the query has no target, so (K) checks nothing /// there. @@ -335,19 +301,21 @@ impl AccuracyRules { /// Whether the selection `edges` (by position in /// `SelectableExecutions::edges`) with the outputs `answering` (query, /// execution answering it, in `SelectableExecutions::answering` order) - /// meets (K): each execution a query with a target reaches, composed - /// ([`Self::compose`]) from the producers selected on its inputs, - /// admits its inputs' errors and, at each output of a query with a - /// target, is within the root's limits. + /// meets (K), exactly in f64: each execution a query with a target + /// reaches, composed ([`Self::compose`]) from the producers selected + /// on its inputs, admits its inputs' errors and, at each output of a + /// query with a target, is within the root's limits. `Err` names the + /// first query (in `answering` order) whose output misses its target. /// /// Example: p99 / p50 at relative 0.03 over two readouts at 0.02 does - /// not meet it (1.111 · 0.04 = 0.044); over two at 0.01 it does. + /// not meet it (1.111 · 0.04 = 0.044), so it is `Err` with the ratio's + /// query; over two at 0.01 it does. pub(super) fn selection_meets_targets( &self, inputs: &SelectionInputs<'_>, edges: &[usize], answering: &[(QueryId, usize)], - ) -> bool { + ) -> Result<(), QueryId> { let selectable = inputs.selectable; let count = selectable.executions().len(); let mut reads: Vec> = vec![Vec::new(); count]; @@ -358,47 +326,48 @@ impl AccuracyRules { } let mut composed: Vec> = vec![None; count]; let mut on_walk = vec![false; count]; - let mut pending: Vec<(usize, bool)> = answering.iter().map(|&(_, v)| (v, false)).collect(); - while let Some((c, producers_done)) = pending.pop() { - if composed[c].is_some() { - continue; - } - if !producers_done { - if on_walk[c] { - // A cycle, which (E) forbids. - return false; - } - on_walk[c] = true; - pending.push((c, true)); - pending.extend( - reads[c] - .iter() - .filter(|&&(_, u)| composed[u].is_none()) - .map(|&(_, u)| (u, false)), - ); - continue; - } - if !self.reached_by_query_with_target[c] { - // No query with a target reaches it: (K) has no rows for - // it, nor for its producers. - composed[c] = Some(ExecutionError::default()); - continue; - } - let Some(inputs_read) = reads[c] - .iter() - .map(|&(i, u)| Some((i, composed[u]?))) - .collect::>>() - else { - return false; - }; - let composition = self.compose(c, inputs_read); - if !composition.inputs_admitted { - return false; - } - composed[c] = Some(composition.error); - } let memo = &inputs.physical.logical_candidates.memo; for (&(_, group), &(query, v)) in memo.output_groups().iter().zip(answering) { + // Compose `v` and every execution below it, producers first. + let mut pending: Vec<(usize, bool)> = vec![(v, false)]; + while let Some((c, producers_done)) = pending.pop() { + if composed[c].is_some() { + continue; + } + if !producers_done { + if on_walk[c] { + // A cycle, which (E) forbids. + return Err(query); + } + on_walk[c] = true; + pending.push((c, true)); + pending.extend( + reads[c] + .iter() + .filter(|&&(_, u)| composed[u].is_none()) + .map(|&(_, u)| (u, false)), + ); + continue; + } + if !self.reached_by_query_with_target[c] { + // No query with a target reaches it: (K) has no rows + // for it, nor for its producers. + composed[c] = Some(ExecutionError::default()); + continue; + } + let Some(inputs_read) = reads[c] + .iter() + .map(|&(i, u)| Some((i, composed[u]?))) + .collect::>>() + else { + return Err(query); + }; + let composition = self.compose(c, inputs_read); + if !composition.inputs_admitted { + return Err(query); + } + composed[c] = Some(composition.error); + } let Some(limits) = self.root_limits(query, group) else { continue; }; @@ -408,17 +377,14 @@ impl AccuracyRules { .iter() .zip(limits.tolerated_error_per_metric) .any(|(&error, limit)| error > limit) + || limits + .tolerated_failure_probability_per_output_value + .is_some_and(|limit| error.failure_probability > limit) { - return false; - } - if limits - .tolerated_failure_probability_per_output_value - .is_some_and(|limit| error.failure_probability > limit) - { - return false; + return Err(query); } } - true + Ok(()) } } diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 073e7feaf..81027786b 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -383,6 +383,15 @@ pub(crate) enum SelectionMilpError { Infeasible, /// The solver failed. Solver(String), + /// The solver's selection misses `query`'s accuracy target when + /// checked exactly: its limits, such as a failure probability of + /// 1e-12 per output value (δ = 0.01 over 1e10 values), are below the + /// solver's feasibility tolerance 1e-10, so (K) cannot be trusted. + TargetBelowSolverPrecision { + query: QueryId, + /// The target's δ per output value; `None` without a δ. + failure_probability_per_output_value: Option, + }, } /// Builds and solves the MILP: minimize @@ -398,7 +407,10 @@ pub(crate) enum SelectionMilpError { /// [`copies_in_equivalent_contexts`]), with /// the $h_v$ of a candidate whose /// executions are all left out, then the coefficients kept are scaled so -/// the largest is 1,000 (HiGHS's tolerances are absolute). +/// the largest is 1,000 (HiGHS's tolerances are absolute). With (K), +/// HiGHS solves at feasibility tolerance 1e-10 and the selection is then +/// checked exactly ([`AccuracyRules::selection_meets_targets`]): one that +/// misses a target is [`SelectionMilpError::TargetBelowSolverPrecision`]. pub(crate) fn solve_selection( inputs: &SelectionInputs<'_>, ) -> Result { @@ -432,6 +444,7 @@ pub(crate) fn solve_selection( ) }) .unwrap_or_default(); + let k_has_rows = !accuracy_rows.is_empty(); // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs @@ -520,13 +533,13 @@ pub(crate) fn solve_selection( model.set_option("mip_abs_gap", 0.0); model.set_option("output_flag", false); model.set_option("threads", 1); - if let Some(tolerance) = rules - .as_ref() - .and_then(|rules| rules.feasibility_tolerance()) - { - // (K)'s limits can be far below HiGHS's absolute default 1e-7. - model.set_option("primal_feasibility_tolerance", tolerance); - model.set_option("mip_feasibility_tolerance", tolerance); + if k_has_rows { + // HiGHS's tolerances are absolute: its default 1e-7 is a tenth of a + // per-value δ of 1e-6 (δ = 0.01 over 1e4 values), so (K) solves at + // 1e-10, the least HiGHS accepts. The selection is checked exactly + // after the solve. + model.set_option("primal_feasibility_tolerance", 1e-10); + model.set_option("mip_feasibility_tolerance", 1e-10); } let construction_ms = started.elapsed().as_millis() as u64; let started = Instant::now(); @@ -573,6 +586,27 @@ pub(crate) fn solve_selection( (*query, execution) }) .collect(); + // (K) holds only to the solver's tolerance, 1e-10: a per-value δ near + // or below it may be missed by orders of magnitude. The exact check + // refuses such a selection rather than return a plan that may not + // meet its target. + if let Some(Err(query)) = rules + .as_ref() + .map(|rules| rules.selection_meets_targets(inputs, &selected_edges, &answering)) + { + let memo = &inputs.physical.logical_candidates.memo; + let memo_query = &memo.queries()[query.0]; + return Err(SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value: memo_query.accuracy_target.as_ref().and_then( + |target| { + target.failure_probability_per_output_value( + memo_query.number_of_values_per_query_evaluation, + ) + }, + ), + }); + } let (cost_per_second_without_request_overhead, retention_ms) = selection_cost_per_second(inputs, &selected, &selected_edges, &answering); Ok(SolvedSelection { diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 92ce6f79f..fa4f557e5 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -1595,23 +1595,26 @@ fn contexts_whose_queries_differ_only_in_values_not_split_by_k_are_copies() { } } -/// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of 1e4 -/// values, 1e-6 per value, over readouts at relative 0.01 failing with -/// 5.4e-7 (cost 1) or 4e-7 (cost 3) each: two at 5.4e-7 fail together with -/// up to 1.08e-6, above 1e-6 by 8e-8, within HiGHS's default feasibility -/// tolerance 1e-7; one of each, 9.4e-7, is within it. (K) solves at 1e-9 -/// (`AccuracyRules::feasibility_tolerance`), so it selects one of each. -#[test] -fn k_bounds_a_per_value_failure_probability_below_the_default_solver_tolerance() { +/// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of +/// `values` values, over readouts at relative 0.01 failing with +/// `failure_probabilities` (cheap at cost 1, dear at cost 3) each: the +/// solved selection, with which readout levels it selects (cheap, dear). +fn solve_ratio_over_values( + values: u64, + failure_probabilities: [f64; 2], +) -> Result<(usize, usize), SelectionMilpError> { let mut memo = empty_memo(); - let levels = [(0.01, 5.4e-7), (0.01, 4e-7)]; + let levels = [ + (0.01, failure_probabilities[0]), + (0.01, failure_probabilities[1]), + ]; let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); memo.add_query(MemoQuery { root: QueryRoot::Operator(ratio), accuracy_target: Some(relative(0.03).failure_probability(0.01)), - number_of_values_per_query_evaluation: Some(10_000), + number_of_values_per_query_evaluation: Some(values), }); let physical = physical(memo.clone(), &every_candidate(&memo)); let mut w = vec![0.0; physical.physical_candidates.len()]; @@ -1620,12 +1623,48 @@ fn k_bounds_a_per_value_failure_probability_below_the_default_solver_tolerance() w[physical_of(&physical, readouts[1])] = 3.0; } let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); - let selected = candidates(&solved.unwrap()); + let selected = candidates(&solved?); let count = |level: usize| { [&p99_readouts, &p50_readouts] .iter() .filter(|readouts| selected.contains(&physical_of(&physical, readouts[level]))) .count() }; - assert_eq!((count(0), count(1)), (1, 1), "{selected:?}"); + Ok((count(0), count(1))) +} + +/// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of 1e4 +/// values, 1e-6 per value, over readouts at relative 0.01 failing with +/// 5.4e-7 (cost 1) or 4e-7 (cost 3) each: two at 5.4e-7 fail together with +/// up to 1.08e-6, above 1e-6 by 8e-8, within HiGHS's default feasibility +/// tolerance 1e-7; one of each, 9.4e-7, is within it. (K) solves at 1e-10, +/// so it selects one of each. +#[test] +fn k_bounds_a_per_value_failure_probability_below_the_default_solver_tolerance() { + assert_eq!(solve_ratio_over_values(10_000, [5.4e-7, 4e-7]), Ok((1, 1))); +} + +/// The same ratio over 1e8 values (1e-10 per value) and 1e10 values +/// (1e-12 per value), with readouts failing at 5.4e-11 / 4e-11 and +/// 5.4e-13 / 4e-13: two cheap readouts miss the limit by 8e-12 and 8e-14, +/// within HiGHS's least feasibility tolerance 1e-10, so the solver selects +/// them. The exact check after the solve finds the miss, and selection +/// fails rather than return a plan that may not meet the target. +#[test] +fn a_per_value_failure_probability_below_the_solver_precision_is_an_error() { + for (values, failure_probabilities, per_value) in [ + (100_000_000, [5.4e-11, 4e-11], 1e-10), + (10_000_000_000, [5.4e-13, 4e-13], 1e-12), + ] { + let solved = solve_ratio_over_values(values, failure_probabilities); + let Err(SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value: Some(delta), + }) = solved + else { + panic!("{values}: {solved:?}"); + }; + assert_eq!(query, QueryId(0)); + assert!((delta - per_value).abs() <= 1e-6 * per_value, "{delta}"); + } } diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 27c344621..765d3332c 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -147,9 +147,11 @@ fn feasible_plan_cost_per_second( continue 'repair; } } - if rules - .is_some_and(|rules| !rules.selection_meets_targets(inputs, &plan.edges, &answering)) - { + if rules.is_some_and(|rules| { + rules + .selection_meets_targets(inputs, &plan.edges, &answering) + .is_err() + }) { return None; } let (cost, _) = diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index 9a8d8bf09..0aedf4547 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -198,6 +198,18 @@ pub enum ErrorSelectingPlan { /// The memo search's MILP solver failed. #[error("MILP solver: {0}")] Solver(String), + /// Query `query`'s accuracy target is below the solver's precision: + /// the MILP's selection misses it when checked exactly, so no plan is + /// returned rather than one that may not meet the target. Example: a + /// failure probability of 1e-12 per output value (δ = 0.01 over 1e10 + /// values), below the solver's feasibility tolerance 1e-10. + #[error( + "query {query}: an accuracy target is below the solver's precision (failure probability per output value {failure_probability_per_output_value:?}); the solver's plan misses it, so no plan is returned" + )] + AccuracyTargetBelowSolverPrecision { + query: usize, + failure_probability_per_output_value: Option, + }, /// The selection MILP implements cost model 1 only (#1095 §3). #[error("the selection MILP does not support {0}")] UnsupportedCostModel(String), diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index d5b4227fb..e56309f88 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -67,7 +67,10 @@ pub enum RootPruneReason { /// evaluable at the required failure probability (an HLL distinct /// count, whose failure probability is unknown), or with no guarantee /// at all (UnivMon's distinct count), read by a query with an accuracy - /// target: its error has no bound. + /// target: its error has no bound. Two more cases: a DDSketch quantile + /// over values of unknown sign (its relative guarantee holds only over + /// values of one sign), and any sketch readout under an exact target + /// (δ 0), which no probabilistic guarantee meets. NoGuarantee { consumer: CandidateForMemoGroupId, input: usize, diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 400599972..e90828dfa 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -423,13 +423,18 @@ above the target. **Solver tolerance.** HiGHS's feasibility tolerances are absolute, 1e-7 by default, while a per-value δ is often far smaller: δ = 0.01 over a whole -result of 1e4 values is 1e-6 per value. With a target, HiGHS's primal and MIP -feasibility tolerances are `min(1e-7, 1e-3 · ℓ)` for the smallest positive -limit `ℓ` a target sets, so a row may pass its limit by at most a thousandth -of the smallest one; where that is not below 1e-7, HiGHS keeps its defaults. -HiGHS accepts no tolerance below 1e-10, so a per-value δ below 1e-7 is -checked only to 1e-10. Scaling (K)'s rows by their limits instead kept the defaults but -solved RangeHeavy-8users about 4× slower. +result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth. Whenever +(K) has rows, HiGHS solves at primal and MIP feasibility tolerance 1e-10, the +least it accepts. A row may still pass its limit by up to 1e-10, which for a +per-value δ near or below 1e-10 can be orders of magnitude: at δ = 1e-12 per +value (δ = 0.01 over 1e10 values) the solver may select readouts failing with +1.08e-12 in all. So after the solve, the selection is checked exactly +(`AccuracyRules::selection_meets_targets`, in f64), and a selection that +misses a target is an error naming the query and its per-value δ ("an +accuracy target is below the solver's precision"); planning never returns a +plan that may not meet its target. Deriving the tolerance from the smallest +limit instead solved RollupHeavy-8users about 2.8× slower, and scaling (K)'s +rows by their limits solved RangeHeavy-8users about 4× slower. **Context copies.** A copy of an equivalent evaluation context shares its original's variables and has no rows of its own. Two contexts are copies only From 9a67a55607db91049df4a74e6a66abdf6f9e02f7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:24:19 +0000 Subject: [PATCH 4/4] fix(stage3): solve at HiGHS's default tolerances and again at 1e-10 only when the exact (K) check fails (#1207 PR 8, review round 3) The selection MILP solves at HiGHS's default feasibility tolerances. The selection is then checked exactly against every accuracy target; one that misses a target is solved once more at primal and MIP feasibility tolerance 1e-10 (LEAST_HIGHS_FEASIBILITY_TOLERANCE) and checked again, and a miss after that is AccuracyTargetBelowSolverPrecision as before. Selection reports milp_solve_count (1 or 2). --- crates/devtools/src/bin/asap_planner.rs | 5 +- crates/stage3-plan-selection/src/lib.rs | 1 + .../src/milp/model/mod.rs | 181 ++++++++++-------- .../src/milp/model/tests.rs | 41 ++-- crates/stage3-plan-selection/src/report.rs | 7 +- docs/design_docs/concepts/accuracy-models.md | 29 +-- docs/develop_docs/library-api.md | 2 +- 7 files changed, 162 insertions(+), 104 deletions(-) diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 7b1b04399..26f6a3108 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -63,7 +63,9 @@ // `PhysicalASAPDAG`) and its `Selection`: `accepted_accuracy_per_query`, // `pruned_candidates`, `milp_objective_cost_per_second` (the selected // plan's cost per second), `milp_variables`, `milp_constraints`, -// `milp_construction_ms` and `milp_solve_ms`; `error` when no plan was +// `milp_construction_ms`, `milp_solve_ms` and `milp_solve_count` (2 when +// the solve at HiGHS's default tolerances missed an accuracy target and +// was repeated at 1e-10); `error` when no plan was // selected. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them; `materialized_views_error` when the @@ -644,6 +646,7 @@ fn selection_json(selection: &Result<(PhysicalASAPDAG, Selection), SelectionErro "milp_constraints": selection.milp_constraint_count, "milp_construction_ms": selection.milp_construction_ms, "milp_solve_ms": selection.milp_solve_ms, + "milp_solve_count": selection.milp_solve_count, }) } diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index c50c77014..d0ab792a1 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -153,6 +153,7 @@ pub fn select_plan( milp_constraint_count: solved.constraint_count, milp_construction_ms: solved.construction_ms, milp_solve_ms: solved.solve_ms, + milp_solve_count: solved.solve_count, }; Ok((plan, selection)) } diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 81027786b..26b7fe166 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -364,7 +364,13 @@ pub(crate) struct SolvedSelection { pub(crate) variable_count: usize, pub(crate) constraint_count: usize, pub(crate) construction_ms: u64, + /// The time of every HiGHS solve, summed. pub(crate) solve_ms: u64, + /// How many times HiGHS solved the MILP: 1, or 2 when the selection at + /// HiGHS's default tolerances missed an accuracy target when checked + /// exactly and the MILP was solved again at + /// [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`]. + pub(crate) solve_count: usize, } /// A selected $y_{\kappa,u,c,i}$: edge `edge` from execution `producer` @@ -386,7 +392,8 @@ pub(crate) enum SelectionMilpError { /// The solver's selection misses `query`'s accuracy target when /// checked exactly: its limits, such as a failure probability of /// 1e-12 per output value (δ = 0.01 over 1e10 values), are below the - /// solver's feasibility tolerance 1e-10, so (K) cannot be trusted. + /// solver's least feasibility tolerance, + /// [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`], so (K) cannot be trusted. TargetBelowSolverPrecision { query: QueryId, /// The target's δ per output value; `None` without a δ. @@ -394,6 +401,13 @@ pub(crate) enum SelectionMilpError { }, } +/// The smallest primal and MIP feasibility tolerance HiGHS accepts, 1e-10 +/// (absolute; HiGHS's default is 1e-7). A second solve uses it when the +/// selection at the default tolerances misses an accuracy target: a +/// per-value δ of 1e-6 (δ = 0.01 over 1e4 values) may be passed by up to +/// 1e-7 at the default, but only by 1e-10 here. +const LEAST_HIGHS_FEASIBILITY_TOLERANCE: f64 = 1e-10; + /// Builds and solves the MILP: minimize /// $\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} /// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} @@ -407,10 +421,12 @@ pub(crate) enum SelectionMilpError { /// [`copies_in_equivalent_contexts`]), with /// the $h_v$ of a candidate whose /// executions are all left out, then the coefficients kept are scaled so -/// the largest is 1,000 (HiGHS's tolerances are absolute). With (K), -/// HiGHS solves at feasibility tolerance 1e-10 and the selection is then -/// checked exactly ([`AccuracyRules::selection_meets_targets`]): one that -/// misses a target is [`SelectionMilpError::TargetBelowSolverPrecision`]. +/// the largest is 1,000 (HiGHS's tolerances are absolute). HiGHS solves +/// at its default tolerances and the selection is then checked exactly +/// ([`AccuracyRules::selection_meets_targets`]); a selection that misses a +/// target is solved once more at [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`], +/// and if that one misses too, the error is +/// [`SelectionMilpError::TargetBelowSolverPrecision`]. pub(crate) fn solve_selection( inputs: &SelectionInputs<'_>, ) -> Result { @@ -444,7 +460,6 @@ pub(crate) fn solve_selection( ) }) .unwrap_or_default(); - let k_has_rows = !accuracy_rows.is_empty(); // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs @@ -533,80 +548,91 @@ pub(crate) fn solve_selection( model.set_option("mip_abs_gap", 0.0); model.set_option("output_flag", false); model.set_option("threads", 1); - if k_has_rows { - // HiGHS's tolerances are absolute: its default 1e-7 is a tenth of a - // per-value δ of 1e-6 (δ = 0.01 over 1e4 values), so (K) solves at - // 1e-10, the least HiGHS accepts. The selection is checked exactly - // after the solve. - model.set_option("primal_feasibility_tolerance", 1e-10); - model.set_option("mip_feasibility_tolerance", 1e-10); - } let construction_ms = started.elapsed().as_millis() as u64; - let started = Instant::now(); - let solved = model - .try_solve() - .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; - let solve_ms = started.elapsed().as_millis() as u64; - use highs::{HighsModelStatus, HighsSolutionStatus}; - let columns: Vec = match solved.status() { - // No constraint: no query reads an operator (literal-only scalar - // roots), so nothing is selected. - HighsModelStatus::ModelEmpty => vec![0.0; variable_count], - HighsModelStatus::Optimal - if solved.primal_solution_status() == HighsSolutionStatus::Feasible => - { - solved.get_solution().columns().to_vec() - } - HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { - return Err(SelectionMilpError::Infeasible) - } - other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), - }; - let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; let selectable = inputs.selectable; let executions = selectable.executions(); - let selected: Vec = (0..variables.x.len()) - .filter(|&index| chosen(variables.x[index])) - .collect(); - let selected_edges: Vec = variables - .y - .iter() - .filter(|&(_, &y)| chosen(y)) - .map(|(&position, _)| position) - .collect(); - let answering: Vec<(QueryId, usize)> = selectable - .answering() - .iter() - .map(|(query, answering)| { - let execution = answering - .iter() - .copied() - .find(|&execution| chosen(variables.z[&(*query, execution)])) - .expect("(C): one output per query output reference"); - (*query, execution) - }) - .collect(); - // (K) holds only to the solver's tolerance, 1e-10: a per-value δ near - // or below it may be missed by orders of magnitude. The exact check - // refuses such a selection rather than return a plan that may not - // meet its target. - if let Some(Err(query)) = rules - .as_ref() - .map(|rules| rules.selection_meets_targets(inputs, &selected_edges, &answering)) - { - let memo = &inputs.physical.logical_candidates.memo; - let memo_query = &memo.queries()[query.0]; - return Err(SelectionMilpError::TargetBelowSolverPrecision { - query, - failure_probability_per_output_value: memo_query.accuracy_target.as_ref().and_then( - |target| { - target.failure_probability_per_output_value( - memo_query.number_of_values_per_query_evaluation, - ) - }, - ), - }); - } + let mut solve_ms = 0; + let mut solve_count = 0; + let (selected, selected_edges, answering) = loop { + let started = Instant::now(); + let solved = model + .try_solve() + .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; + solve_ms += started.elapsed().as_millis() as u64; + solve_count += 1; + use highs::{HighsModelStatus, HighsSolutionStatus}; + let columns: Vec = match solved.status() { + // No constraint: no query reads an operator (literal-only scalar + // roots), so nothing is selected. + HighsModelStatus::ModelEmpty => vec![0.0; variable_count], + HighsModelStatus::Optimal + if solved.primal_solution_status() == HighsSolutionStatus::Feasible => + { + solved.get_solution().columns().to_vec() + } + HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { + return Err(SelectionMilpError::Infeasible) + } + other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), + }; + let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; + let selected: Vec = (0..variables.x.len()) + .filter(|&index| chosen(variables.x[index])) + .collect(); + let selected_edges: Vec = variables + .y + .iter() + .filter(|&(_, &y)| chosen(y)) + .map(|(&position, _)| position) + .collect(); + let answering: Vec<(QueryId, usize)> = selectable + .answering() + .iter() + .map(|(query, answering)| { + let execution = answering + .iter() + .copied() + .find(|&execution| chosen(variables.z[&(*query, execution)])) + .expect("(C): one output per query output reference"); + (*query, execution) + }) + .collect(); + // (K) holds only to the solver's tolerance: at the default 1e-7, a + // per-value δ of 1e-6 may be missed by a tenth, and at 1e-10 a δ + // near or below 1e-10 by orders of magnitude. The exact check sends + // a selection that misses a target to a second solve at + // LEAST_HIGHS_FEASIBILITY_TOLERANCE, and refuses one that misses + // after it rather than return a plan that may not meet its target. + let Some(Err(query)) = rules + .as_ref() + .map(|rules| rules.selection_meets_targets(inputs, &selected_edges, &answering)) + else { + break (selected, selected_edges, answering); + }; + if solve_count == 2 { + let memo = &inputs.physical.logical_candidates.memo; + let memo_query = &memo.queries()[query.0]; + return Err(SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_per_output_value: memo_query.accuracy_target.as_ref().and_then( + |target| { + target.failure_probability_per_output_value( + memo_query.number_of_values_per_query_evaluation, + ) + }, + ), + }); + } + model = solved.into(); + model.set_option( + "primal_feasibility_tolerance", + LEAST_HIGHS_FEASIBILITY_TOLERANCE, + ); + model.set_option( + "mip_feasibility_tolerance", + LEAST_HIGHS_FEASIBILITY_TOLERANCE, + ); + }; let (cost_per_second_without_request_overhead, retention_ms) = selection_cost_per_second(inputs, &selected, &selected_edges, &answering); Ok(SolvedSelection { @@ -633,6 +659,7 @@ pub(crate) fn solve_selection( constraint_count, construction_ms, solve_ms, + solve_count, }) } diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index fa4f557e5..ee8b8170e 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -1597,12 +1597,13 @@ fn contexts_whose_queries_differ_only_in_values_not_split_by_k_are_copies() { /// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of /// `values` values, over readouts at relative 0.01 failing with -/// `failure_probabilities` (cheap at cost 1, dear at cost 3) each: the -/// solved selection, with which readout levels it selects (cheap, dear). +/// `failure_probabilities` (cheap at cost 1, dear at cost 3) each: how +/// many readouts of each level the selection reads (cheap, dear) and how +/// many times HiGHS solved the MILP. fn solve_ratio_over_values( values: u64, failure_probabilities: [f64; 2], -) -> Result<(usize, usize), SelectionMilpError> { +) -> Result<(usize, usize, usize), SelectionMilpError> { let mut memo = empty_memo(); let levels = [ (0.01, failure_probabilities[0]), @@ -1623,33 +1624,49 @@ fn solve_ratio_over_values( w[physical_of(&physical, readouts[1])] = 3.0; } let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); - let selected = candidates(&solved?); + let solved = solved?; + let selected = candidates(&solved); let count = |level: usize| { [&p99_readouts, &p50_readouts] .iter() .filter(|readouts| selected.contains(&physical_of(&physical, readouts[level]))) .count() }; - Ok((count(0), count(1))) + Ok((count(0), count(1), solved.solve_count)) } /// p99 / p50 at relative 0.03 with δ = 0.01 over a whole result of 1e4 /// values, 1e-6 per value, over readouts at relative 0.01 failing with /// 5.4e-7 (cost 1) or 4e-7 (cost 3) each: two at 5.4e-7 fail together with /// up to 1.08e-6, above 1e-6 by 8e-8, within HiGHS's default feasibility -/// tolerance 1e-7; one of each, 9.4e-7, is within it. (K) solves at 1e-10, -/// so it selects one of each. +/// tolerance 1e-7, so the first solve selects the two cheap readouts. The +/// exact check finds the miss, and the second solve, at tolerance 1e-10, +/// selects one of each (9.4e-7, within the limit). +#[test] +fn a_selection_missing_its_target_at_the_default_tolerance_is_solved_again_at_the_least_tolerance() +{ + assert_eq!( + solve_ratio_over_values(10_000, [5.4e-7, 4e-7]), + Ok((1, 1, 2)) + ); +} + +/// The same ratio over readouts failing with 4e-7 (cost 1) or 3e-7 +/// (cost 3) each: two cheap readouts fail with up to 8e-7, within 1e-6, +/// so the solve at HiGHS's default tolerances meets the target and is +/// the only solve. #[test] -fn k_bounds_a_per_value_failure_probability_below_the_default_solver_tolerance() { - assert_eq!(solve_ratio_over_values(10_000, [5.4e-7, 4e-7]), Ok((1, 1))); +fn a_selection_meeting_its_target_at_the_default_tolerance_is_solved_once() { + assert_eq!(solve_ratio_over_values(10_000, [4e-7, 3e-7]), Ok((2, 0, 1))); } /// The same ratio over 1e8 values (1e-10 per value) and 1e10 values /// (1e-12 per value), with readouts failing at 5.4e-11 / 4e-11 and /// 5.4e-13 / 4e-13: two cheap readouts miss the limit by 8e-12 and 8e-14, -/// within HiGHS's least feasibility tolerance 1e-10, so the solver selects -/// them. The exact check after the solve finds the miss, and selection -/// fails rather than return a plan that may not meet the target. +/// within HiGHS's least feasibility tolerance 1e-10, so both solves, at +/// the default tolerances and again at 1e-10, select them. The exact check +/// after the second solve finds the miss, and selection fails rather than +/// return a plan that may not meet the target. #[test] fn a_per_value_failure_probability_below_the_solver_precision_is_an_error() { for (values, failure_probabilities, per_value) in [ diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index 0aedf4547..4234483d9 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -167,9 +167,14 @@ pub struct Selection { pub milp_variable_count: usize, /// The selection MILP's constraint rows. pub milp_constraint_count: usize, - /// How long building and solving the selection MILP took. + /// How long building and solving the selection MILP took, the solve + /// time summed over `milp_solve_count` solves. pub milp_construction_ms: u64, pub milp_solve_ms: u64, + /// How many times HiGHS solved the MILP: 1, or 2 when the selection at + /// HiGHS's default feasibility tolerances (1e-7) missed an accuracy + /// target when checked exactly and was solved again at 1e-10. + pub milp_solve_count: usize, } /// Why Stage 3 selected no plan. diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index e90828dfa..c698b99cf 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -423,18 +423,23 @@ above the target. **Solver tolerance.** HiGHS's feasibility tolerances are absolute, 1e-7 by default, while a per-value δ is often far smaller: δ = 0.01 over a whole -result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth. Whenever -(K) has rows, HiGHS solves at primal and MIP feasibility tolerance 1e-10, the -least it accepts. A row may still pass its limit by up to 1e-10, which for a -per-value δ near or below 1e-10 can be orders of magnitude: at δ = 1e-12 per -value (δ = 0.01 over 1e10 values) the solver may select readouts failing with -1.08e-12 in all. So after the solve, the selection is checked exactly -(`AccuracyRules::selection_meets_targets`, in f64), and a selection that -misses a target is an error naming the query and its per-value δ ("an -accuracy target is below the solver's precision"); planning never returns a -plan that may not meet its target. Deriving the tolerance from the smallest -limit instead solved RollupHeavy-8users about 2.8× slower, and scaling (K)'s -rows by their limits solved RangeHeavy-8users about 4× slower. +result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth: two +readouts failing with 5.4e-7 each compose to 1.08e-6 and pass that limit +within the default tolerance. So HiGHS first solves at its defaults, and the +selection is then checked exactly (`AccuracyRules::selection_meets_targets`, +in f64). A selection that misses a target is solved once more at primal and +MIP feasibility tolerance 1e-10, the least HiGHS accepts, and checked again. +A row may still pass its limit by up to 1e-10, which for a per-value δ near +or below 1e-10 can be orders of magnitude: at δ = 1e-12 per value (δ = 0.01 +over 1e10 values) the solver may select readouts failing with 1.08e-12 in +all. A selection that misses a target after the second solve is an error +naming the query and its per-value δ ("an accuracy target is below the +solver's precision"); planning never returns a plan that may not meet its +target. The stricter tolerance is paid for only when the check needs it: +solving every (K) at 1e-10 made RangeHeavy-8users about 6× slower, deriving +the tolerance from the smallest limit solved RollupHeavy-8users about 2.8× +slower, and scaling (K)'s rows by their limits solved RangeHeavy-8users +about 4× slower. **Context copies.** A copy of an equivalent evaluation context shares its original's variables and has no rows of its own. Two contexts are copies only diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 6e18189ca..7d3e11dac 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -202,7 +202,7 @@ asap_stage3_plan_selection::select_plan( | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory}` chooses among the candidates that fit | | `PlanOutput.plan` | The selected plan, one `PhysicalASAPDAG`: `roots` answer the operator queries and `scalar_roots` the standalone scalar queries, in `QueryWorkload::entries()` order; each materialized node carries its `retention_ms`. Its cost is `asap_cost::plan_cost` of it | -| `PlanningRun` | `pass1_memo`, `logical_asap_candidates`, `physical_asap_candidates`, `selection` (the plan and Stage 3's `Selection`: `accepted_accuracy_per_query`, `pruned_candidates`, `milp_variables`, `milp_constraints`, `milp_construction_ms`, `milp_solve_ms`) and `report` (Pass 2's and Stage 2's counts and each stage's time); `pass1_memo` is kept only with `PlanSearchSettings::inspect` | +| `PlanningRun` | `pass1_memo`, `logical_asap_candidates`, `physical_asap_candidates`, `selection` (the plan and Stage 3's `Selection`: `accepted_accuracy_per_query`, `pruned_candidates`, `milp_variables`, `milp_constraints`, `milp_construction_ms`, `milp_solve_ms`, `milp_solve_count`) and `report` (Pass 2's and Stage 2's counts and each stage's time); `pass1_memo` is kept only with `PlanSearchSettings::inspect` | Each candidate's `ResourceUsage` holds its usage without prices: `cpu_ops_per_second`, `scanned_bytes_per_second`, `average_memory_bytes`,