diff --git a/crates/accuracy/src/estimate_dependence.rs b/crates/accuracy/src/estimate_dependence.rs new file mode 100644 index 000000000..28d7192fe --- /dev/null +++ b/crates/accuracy/src/estimate_dependence.rs @@ -0,0 +1,136 @@ +//! Which estimates an output value depends on (`accuracy-models.md`, +//! "Which estimates a result depends on"): the operator classes and the +//! "error reaches the output" test that Pass 2's failure-probability split +//! (`readout_caps`) and Stage 3's constraint (K) both count by, so the +//! level Pass 2 gives a query meets (K) for that query. +//! +//! An estimate is one sketch readout value in one evaluation. An output +//! value depends on one value of each input an operator reads value by +//! value ([`inputs_read_value_by_value`]), on one input's value when the +//! operator copies each value from one input +//! ([`takes_each_value_from_one_input`]), and on every value of any other +//! input; only through inputs whose error reaches the output +//! ([`input_error_reaches_output`]) or whose values are chosen by +//! estimates ([`values_chosen_by_estimate`]); both read through +//! `ErrorMetricsOfMemo::input_estimates_reach_output`. + +use asap_types::ir::operator::arguments::BinaryOpKind; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::SketchStatistic; + +use crate::metric_rule::ErrorContractPerMetric; + +/// Whether input `input` of a candidate with the error contracts +/// `contract_per_metric` can make its output miss: its error reaches the +/// output in some metric ($\beta > 0$). A count's input and the right +/// side of `and` and `unless` cannot ($\beta = 0$: only which series or +/// rows they hold is read), nor any input of an operator with no contract +/// (a subquery's operand, which may not carry error at all). +pub(crate) fn input_error_reaches_output( + contract_per_metric: &ErrorContractPerMetric, + input: usize, +) -> bool { + contract_per_metric + .iter() + .flatten() + .any(|contract| contract.input_error_reaches_output(input)) +} + +/// Per group of `memo`, by id: whether which values its output holds is +/// chosen by estimates, so that a consumer keeping or dropping a value by +/// its labels depends on every estimate that choice read. A top-k readout +/// chooses its keys by all the estimates it ranks +/// ([`ranks_by_estimate`]), and an operator reading such an input value by +/// value ([`inputs_read_value_by_value`]) passes the choice on: `topk(3, +/// q) * 2` holds the 3 keys the top-k chose. A reduction does not: `sum` +/// returns its one value whichever keys it reads. `order` is the memo's +/// topological order (`Memo::topological_order`). +/// +/// Example: in `p90 and topk(3, q)`, the top-k's group and the `and`'s +/// are chosen by estimate, `p90`'s is not. +pub fn values_chosen_by_estimate(memo: &Memo, order: &[MemoGroupId]) -> Vec { + let mut chosen = vec![false; memo.groups().len()]; + for &group in order { + chosen[group.0 as usize] = + memo.groups()[group.0 as usize] + .candidates + .iter() + .any(|candidate| { + let operator = &candidate.expression.operator; + ranks_by_estimate(operator) + || inputs_read_value_by_value(operator).is_some_and(|value_by_value| { + operator + .children() + .iter() + .take(value_by_value) + .any(|child| chosen[child.0 as usize]) + }) + }); + } + chosen +} + +/// Whether `operator`, a readout, chooses its output values by the +/// estimates it computes: a top-k ranks every item of its state, so each +/// of its output values depends on all of them. +pub fn ranks_by_estimate(operator: &Operator) -> bool { + matches!( + operator, + Operator::ASAP(ASAPOp::SummaryEstimate { + query: SketchStatistic::TopK { .. }, + .. + }) + ) +} + +/// How many direct inputs `operator` reads value by value: each output +/// value from one value (row or series) of each of them; `None` for an +/// operator one of whose output values may read many values of an input +/// (an aggregate, a window function, a time range or subquery, a `limit` +/// choosing rows by value, a deduplication). A filter, projection, sort, +/// time shift, relabel, label enrichment or series sample: 1, each output +/// value one input value. A binary operator, join or relational set +/// operator: 2, one value of each side; a concatenation: each of its +/// inputs. Operators its scalar expressions reference (`scalar(r)`) follow +/// the direct inputs and are read whole. Both counts are per evaluation, +/// not per subquery step: `max_over_time(q[1h:1m])` reads each readout +/// value of `q` at 60 steps, which they do not multiply in (no cap passes +/// through a subquery to the readouts below it). +pub fn inputs_read_value_by_value(operator: &Operator) -> Option { + match operator { + Operator::NonASAP( + NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. }, + ) => Some(1), + Operator::NonASAP( + NonASAPOp::BinaryOp { .. } | NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. }, + ) => Some(2), + Operator::NonASAP(NonASAPOp::Concat { children }) => Some(children.len()), + _ => None, + } +} + +/// Whether `operator` copies each output value from exactly one of its +/// inputs: a PromQL set operator (`q or r` takes a series' value from `q` +/// when `q` has the series, else from `r`; `and` and `unless` only from +/// the left) or a concatenation (each row from one of its inputs). +pub fn takes_each_value_from_one_input(operator: &Operator) -> bool { + matches!( + operator, + Operator::NonASAP( + NonASAPOp::BinaryOp { + operator: BinaryOperator { + kind: BinaryOpKind::Set(_), + .. + }, + .. + } | NonASAPOp::Concat { .. } + ) + ) +} diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 075ac794c..75dd89a41 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -18,6 +18,7 @@ //! can use it. mod error_contract; +mod estimate_dependence; mod evidence; mod guarantee; mod logical_edge; @@ -27,6 +28,9 @@ mod readout; mod readout_caps; pub use error_contract::ErrorContract; +pub use estimate_dependence::{ + inputs_read_value_by_value, ranks_by_estimate, takes_each_value_from_one_input, +}; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::{tolerated_limit, within_limit}; @@ -35,4 +39,4 @@ pub use measured::{ MeasuredSketchAccuracyError, }; pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; -pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; +pub use readout_caps::{readout_levels, ReadoutLevels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 6e947e0e4..0d01e9782 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -12,6 +12,7 @@ use asap_types::ir::{QueryRoot, ScalarExpr}; use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId}; use crate::error_contract::{error_contract, ErrorContract}; +use crate::estimate_dependence::{takes_each_value_from_one_input, values_chosen_by_estimate}; use crate::readout::sketch_readout_error_metric; use crate::readout_caps::caps; @@ -33,6 +34,9 @@ pub struct ErrorMetricsOfMemo { /// not a sketch readout, or one whose summary type has no guarantee /// for the statistic it reads. readout_error_metric: Vec>>, + /// [group]: whether which values its output holds is chosen by + /// estimates ([`values_chosen_by_estimate`]). + values_chosen_by_estimate: Vec, } impl ErrorMetricsOfMemo { @@ -91,6 +95,7 @@ impl ErrorMetricsOfMemo { error_contract, error_metrics_allowed_at_output, readout_error_metric, + values_chosen_by_estimate: values_chosen_by_estimate(memo, order), } } @@ -104,6 +109,50 @@ impl ErrorMetricsOfMemo { .as_ref() } + /// Whether an estimate input `input` of `candidate` (of `memo`) + /// depends on can make the output miss, so the output's failure + /// probability counts it: the input's error reaches the output in some + /// metric ($\beta > 0$, + /// [`crate::estimate_dependence::input_error_reaches_output`]), or which + /// values the input holds is chosen by estimates + /// ([`values_chosen_by_estimate`]). Pass 2's failure-probability split + /// and Stage 3's constraint (K) both count by it. + /// + /// Example: in `p90 and r` only `p90`'s estimates count; in `p90 and + /// topk(3, q)` the top-k's do too, as they choose the keys `and` keeps. + pub fn input_estimates_reach_output( + &self, + memo: &Memo, + candidate: CandidateForMemoGroupId, + input: usize, + ) -> bool { + let child = memo.candidate(candidate).expression.operator.children()[input]; + self.values_chosen_by_estimate[child.0 as usize] + || crate::estimate_dependence::input_error_reaches_output( + &self.error_contract[candidate.group.0 as usize][candidate.index_in_group], + input, + ) + } + + /// Whether each output value of `candidate` (of `memo`) depends on the + /// estimates of one input only: it copies each value from one input + /// ([`takes_each_value_from_one_input`]) and no input's values are + /// chosen by estimates. `q or r` takes `r`'s value only where `q` has + /// no such series, so with `q` a top-k that choice reads all of `q`'s + /// estimates. + pub fn each_value_depends_on_one_input( + &self, + memo: &Memo, + candidate: CandidateForMemoGroupId, + ) -> bool { + let operator = &memo.candidate(candidate).expression.operator; + takes_each_value_from_one_input(operator) + && !operator + .children() + .iter() + .any(|child| self.values_chosen_by_estimate[child.0 as usize]) + } + /// The error metric of `candidate` when it is a sketch readout. pub(crate) fn readout_error_metric( &self, diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index 6ada0ba8d..31264f6ee 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -6,12 +6,15 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use asap_types::accuracy::{AccuracyMetric, AccuracyTarget, ReadoutValueCounts}; -use asap_types::ir::operator::arguments::BinaryOpKind; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; +use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId, RequiredAccuracy}; -use asap_types::summary_type::{parameter_list, SketchStatistic}; +use asap_types::summary_type::parameter_list; +use crate::estimate_dependence::{ + input_error_reaches_output, inputs_read_value_by_value, ranks_by_estimate, + takes_each_value_from_one_input, +}; use crate::logical_edge::build_meets_level; use crate::metric_rule::{query_roots_with_target, ErrorContractPerMetric}; use crate::readout::sketch_readout; @@ -58,6 +61,23 @@ pub struct ReadoutRequiredAccuracy { pub queries_without_per_value_failure_probability: Vec, } +/// What [`readout_levels`] gives: each readout's levels, and the readout +/// values each query's δ is split over. +#[derive(Debug, Clone, PartialEq)] +pub struct ReadoutLevels { + /// Per sketch readout without a requirement yet: its levels. + pub of_readout: BTreeMap, + /// Per query, by index: the readout values (estimates) its output can + /// depend on, `R` and the fan-in + /// ([`ReadoutsReadByQueries::readout_values`]), which name the bound + /// its δ is split by (`AccuracyTarget::failure_probability_bound`) and + /// Stage 3's constraint (K) checks; `None` for a query without a + /// target. + /// + /// Example: `q or r` over 100 series each: `R` 200, fan-in 1. + pub readout_values_of_query: Vec>, +} + /// The required-accuracy levels of the sketch readouts without a /// requirement yet, per readout ([`ReadoutRequiredAccuracy`]), from /// `error_metrics` of `memo` ([`ErrorMetricsOfMemo::of`]). `order` is the @@ -93,7 +113,7 @@ pub fn readout_levels( error_metrics: &ErrorMetricsOfMemo, order: &[MemoGroupId], evidence: &dyn AccuracyEvidence, -) -> BTreeMap { +) -> ReadoutLevels { let caps = caps(memo, &error_metrics.error_contract, order); let mut readouts: Vec<(CandidateForMemoGroupId, AccuracyMetric)> = Vec::new(); for (index, group) in memo.groups().iter().enumerate() { @@ -113,10 +133,7 @@ pub fn readout_levels( } } } - let readout_groups: BTreeSet = - readouts.iter().map(|(readout, _)| readout.group).collect(); - let readouts_read = - readouts_read_by_queries(memo, &error_metrics.error_contract, order, &readout_groups); + let readouts_read = readouts_read_by_queries(memo, error_metrics, order); let mut readouts_of_group: HashMap< (MemoGroupId, AccuracyMetric), Vec, @@ -129,7 +146,10 @@ pub fn readout_levels( } let mut levels_of_group: HashMap<(MemoGroupId, AccuracyMetric), ReadoutRequiredAccuracy> = HashMap::new(); - readouts + let readout_values_of_query = (0..memo.queries().len()) + .map(|query| readouts_read.readout_values.get(&query).copied()) + .collect(); + let of_readout = readouts .into_iter() .map(|(readout, metric)| { let group = readout.group; @@ -148,7 +168,11 @@ pub fn readout_levels( .clone(); (readout, levels) }) - .collect() + .collect(); + ReadoutLevels { + of_readout, + readout_values_of_query, + } } /// The levels of readout group `group` in `metric` ([`readout_levels`]). @@ -409,16 +433,20 @@ struct ReadoutsReadByQueries { /// single level of its cap. Readouts of one value by different summary /// types are alternatives in one group's candidates, not a combination. composed: BTreeSet, - /// Per query, by index: the readout values its output can depend on - /// through β > 0, among alternative candidates of a group the largest. - /// Per evaluation, summed over the readout groups below it - /// (`Memo::max_values_per_evaluation` each); per output value, 1 at a - /// readout (all of its values for a top-k, which ranks them), carried - /// value by value through [`inputs_read_value_by_value`] operators - /// (summed over inputs, or the largest for `or` and a concatenation) - /// and all of the inputs' per-evaluation values through any other - /// operator (a reduction); a readout group also counts its other - /// candidates (a reduction over readouts below it). `max by (job) + /// Per query, by index: the readout values (estimates) its output can + /// depend on, through inputs whose estimates reach their consumer's + /// output (`ErrorMetricsOfMemo::input_estimates_reach_output`: β > 0, + /// or values chosen by estimates), among alternative candidates of a + /// group the largest. Per evaluation, summed over the readout groups + /// below it (`Memo::max_estimates_per_evaluation` each: its values, or + /// every item a top-k ranks); per output value, 1 at a readout (all of + /// its estimates for a top-k, which ranks them), carried value by + /// value through [`inputs_read_value_by_value`] operators (summed over + /// inputs, or the largest where each value depends on one input, + /// `ErrorMetricsOfMemo::each_value_depends_on_one_input`: `or` and a + /// concatenation) and all of the inputs' per-evaluation values through + /// any other operator (a reduction); a readout group also counts its + /// other candidates (a reduction over readouts below it). `max by (job) /// (q{host="h0"} or q{host="h1"})` with 3 series per side: 6 and 6; /// `q or r` alone: 6 and 1; `q / count(r)`: `q`'s alone. A readout /// group reached on two paths (`q or q`) counts twice, which only @@ -427,13 +455,30 @@ struct ReadoutsReadByQueries { } /// The [`ReadoutsReadByQueries`] of `memo`, in one walk up `order` (the -/// memo's topological order) from the readout groups `readout_groups`. +/// memo's topological order) from its readout groups (those with a +/// candidate that is a sketch readout, `ErrorMetricsOfMemo::readout_error_metric`). fn readouts_read_by_queries( memo: &Memo, - error_contract: &[Vec], + error_metrics: &ErrorMetricsOfMemo, order: &[MemoGroupId], - readout_groups: &BTreeSet, ) -> ReadoutsReadByQueries { + let error_contract = &error_metrics.error_contract; + let readout_groups: BTreeSet = memo + .groups() + .iter() + .enumerate() + .filter(|(group, memo_group)| { + (0..memo_group.candidates.len()).any(|index_in_group| { + error_metrics + .readout_error_metric(CandidateForMemoGroupId { + group: MemoGroupId(*group as u32), + index_in_group, + }) + .is_some() + }) + }) + .map(|(group, _)| MemoGroupId(group as u32)) + .collect(); // Per group: the readout values its output can depend on (`Some(0)` // when none, so it carries no readout error; `None` unknown). let none_read = ReadoutValueCounts { @@ -446,24 +491,32 @@ fn readouts_read_by_queries( let candidates = &memo.groups()[group.0 as usize].candidates; let mut most_inputs_reading_with_error = 0; let mut most_readout_values = none_read; - for (candidate, contract_per_metric) in - candidates.iter().zip(&error_contract[group.0 as usize]) + for (index_in_group, (candidate, contract_per_metric)) in candidates + .iter() + .zip(&error_contract[group.0 as usize]) + .enumerate() { + let id = CandidateForMemoGroupId { + group, + index_in_group, + }; let operator = &candidate.expression.operator; let children = operator.children(); - let takes_each_value_from_one_input = takes_each_value_from_one_input(operator); + let reads_readouts = |input: usize| { + readout_values_per_group[children[input].0 as usize].per_evaluation != Some(0) + }; let reads_with_error = |input: &usize| { - readout_values_per_group[children[*input].0 as usize].per_evaluation != Some(0) - && contract_per_metric - .iter() - .flatten() - .any(|contract| contract.error_factor_per_input[*input] > 0.0) + reads_readouts(*input) && input_error_reaches_output(contract_per_metric, *input) + }; + let reads_estimates = |input: &usize| { + reads_readouts(*input) + && error_metrics.input_estimates_reach_output(memo, id, *input) }; let inputs_reading_with_error = (0..children.len()).filter(reads_with_error).count(); // Each output value comes from one input, so the output's error // is one input's, never a sum of several; its values still // depend on the readouts of every input. - let inputs_combined = if takes_each_value_from_one_input { + let inputs_combined = if takes_each_value_from_one_input(operator) { inputs_reading_with_error.min(1) } else { inputs_reading_with_error @@ -471,7 +524,7 @@ fn readouts_read_by_queries( most_inputs_reading_with_error = most_inputs_reading_with_error.max(inputs_combined); let read_of = |input: usize| readout_values_per_group[children[input].0 as usize]; let per_evaluation = (0..children.len()) - .filter(reads_with_error) + .filter(reads_estimates) .try_fold(0u64, |total, input| { total.checked_add(read_of(input).per_evaluation?) }); @@ -479,14 +532,14 @@ fn readouts_read_by_queries( // `value_by_value` inputs, and every value of any other input. let per_output_value = inputs_read_value_by_value(operator).map_or(per_evaluation, |value_by_value| { - let mut per_input = (0..children.len()).filter(reads_with_error).map(|input| { + let mut per_input = (0..children.len()).filter(reads_estimates).map(|input| { if input < value_by_value { read_of(input).per_output_value } else { read_of(input).per_evaluation } }); - if takes_each_value_from_one_input { + if error_metrics.each_value_depends_on_one_input(memo, id) { per_input.try_fold(0u64, |most, values| Some(most.max(values?))) } else { per_input.try_fold(0u64, |total, values| total.checked_add(values?)) @@ -510,16 +563,10 @@ fn readouts_read_by_queries( // output keys are chosen by their estimates; any other candidate of // the group (a reduction over readouts below it) still counts. readout_values_per_group[group.0 as usize] = if readout_groups.contains(&group) { - let per_evaluation = memo.max_values_per_evaluation(group); - let ranks_by_estimate = candidates.iter().any(|candidate| { - matches!( - candidate.expression.operator, - Operator::ASAP(ASAPOp::SummaryEstimate { - query: SketchStatistic::TopK { .. }, - .. - }) - ) - }); + let per_evaluation = memo.max_estimates_per_evaluation(group); + let ranks_by_estimate = candidates + .iter() + .any(|candidate| ranks_by_estimate(&candidate.expression.operator)); let own = ReadoutValueCounts { per_evaluation, per_output_value: if ranks_by_estimate { @@ -546,57 +593,6 @@ fn readouts_read_by_queries( } } -/// How many direct inputs `operator` reads value by value: each output -/// value from one value (row or series) of each of them; `None` for an -/// operator one of whose output values may read many values of an input -/// (an aggregate, a window function, a time range or subquery, a `limit` -/// choosing rows by value, a deduplication). A filter, projection, sort, -/// time shift, relabel, label enrichment or series sample: 1, each output -/// value one input value. A binary operator, join or relational set -/// operator: 2, one value of each side; a concatenation: each of its -/// inputs. Operators its scalar expressions reference (`scalar(r)`) follow -/// the direct inputs and are read whole. Both counts are per evaluation, -/// not per subquery step: `max_over_time(q[1h:1m])` reads each readout -/// value of `q` at 60 steps, which they do not multiply in (no cap passes -/// through a subquery to the readouts below it). -fn inputs_read_value_by_value(operator: &Operator) -> Option { - match operator { - Operator::NonASAP( - NonASAPOp::Filter { .. } - | NonASAPOp::Project { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::TimeShift { .. } - | NonASAPOp::PromqlRelabel { .. } - | NonASAPOp::PromqlInfoEnrich { .. } - | NonASAPOp::PromqlSeriesSample { .. }, - ) => Some(1), - Operator::NonASAP( - NonASAPOp::BinaryOp { .. } | NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. }, - ) => Some(2), - Operator::NonASAP(NonASAPOp::Concat { children }) => Some(children.len()), - _ => None, - } -} - -/// Whether `operator` copies each output value from exactly one of its -/// inputs: a PromQL set operator (`q or r` takes a series' value from `q` -/// when `q` has the series, else from `r`; `and` and `unless` only from -/// the left) or a concatenation (each row from one of its inputs). -fn takes_each_value_from_one_input(operator: &Operator) -> bool { - matches!( - operator, - Operator::NonASAP( - NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Set(_), - .. - }, - .. - } | NonASAPOp::Concat { .. } - ) - ) -} - /// Keeps the larger of `cap` and the known cap of `query` in `metric`. fn raise( caps: &mut HashMap>, diff --git a/crates/accuracy/src/readout_caps/tests.rs b/crates/accuracy/src/readout_caps/tests.rs index 47f49a058..ed846c6f1 100644 --- a/crates/accuracy/src/readout_caps/tests.rs +++ b/crates/accuracy/src/readout_caps/tests.rs @@ -4,7 +4,7 @@ use asap_types::accuracy::AccuracyScope; use asap_types::ir::operator::arguments::{PromQLVectorSetOpKind, Reduction}; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::QueryRoot; +use asap_types::ir::{NonASAPOp, QueryRoot}; use asap_types::stages::{MemoQuery, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use asap_types::test_support::memo_builders::{ @@ -23,6 +23,7 @@ fn levels_of(memo: &Memo, readout: MemoGroupId) -> Vec { &order, &NoMeasuredGuarantees, ) + .of_readout .remove(&own_candidate(readout)) .expect("a sketch readout") .levels @@ -337,6 +338,7 @@ fn failure_probability_of(memo: &Memo, readout: MemoGroupId) -> Option { &order, &NoMeasuredGuarantees, ) + .of_readout .remove(&own_candidate(readout)) .expect("a sketch readout"); if !required @@ -616,12 +618,7 @@ fn a_per_value_ratio_splits_over_the_two_readout_values_of_each_output_value() { with_query(&mut memo, ratio, target.clone(), Some(8)); let order = memo.topological_order().expect("acyclic"); let error_metrics = ErrorMetricsOfMemo::of(&memo, &order); - let read = readouts_read_by_queries( - &memo, - &error_metrics.error_contract, - &order, - &[p90, p50].into(), - ); + let read = readouts_read_by_queries(&memo, &error_metrics, &order); assert!(read.composed.contains(&0)); let counts = read.readout_values[&0]; assert_eq!( diff --git a/crates/integration-tests/tests/accuracy_at_query_roots.rs b/crates/integration-tests/tests/accuracy_at_query_roots.rs index 3b8292de6..3f1b6bd75 100644 --- a/crates/integration-tests/tests/accuracy_at_query_roots.rs +++ b/crates/integration-tests/tests/accuracy_at_query_roots.rs @@ -4,7 +4,8 @@ //! error 0.01, and B, the ratio of its p99 and p50 at relative error 0.03. use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; -use asap_planner::{DeploymentCapabilities, PlanningModels}; +use asap_planner::{DeploymentCapabilities, PlanningModels, PlanningRun}; +use asap_types::ir::{ASAPOp, Operator}; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::summary_type::SketchParams; use asap_types::workload::PlanningWorkload; @@ -16,6 +17,16 @@ const LATENCY_SERIES: usize = 1_000; /// `latency` (`series_count` series, one sample per second each, /// ingested, values declared non-negative). fn workload(queries: &[(&str, &str)], series_count: usize) -> PlanningWorkload { + workload_with_sources(queries, series_count, "") +} + +/// [`workload`] with `more_sources`, JSON members of `sources` after +/// `latency`'s (each starting with a comma). +fn workload_with_sources( + queries: &[(&str, &str)], + series_count: usize, + more_sources: &str, +) -> PlanningWorkload { let entries: Vec = queries .iter() .map(|(query, accuracy)| { @@ -29,7 +40,7 @@ fn workload(queries: &[(&str, &str)], series_count: usize) -> PlanningWorkload { r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": {series_count}.0}}}}, "sources": {{"latency": {{"series_count": {series_count}, "metric_type": "gauge", "sample_interval": 1000, - "column_signs": {{"value": "non_negative"}}}}}}}}}}"#, + "column_signs": {{"value": "non_negative"}}}}{more_sources}}}}}}}"#, entries.join(",") )) .unwrap() @@ -52,7 +63,19 @@ fn selection( series_count: usize, ignore_accuracy: bool, ) -> (PhysicalASAPDAG, Vec) { - let workload = workload(queries, series_count); + let run = planning_run(workload(queries, series_count), ignore_accuracy); + let (plan, selection) = run.selection.as_ref().expect("a plan is selected"); + let accepted = selection + .accepted_accuracy_per_query + .iter() + .map(ToString::to_string) + .collect(); + (plan.clone(), accepted) +} + +/// Every planning stage over `workload`, no raw data kept, with accuracy +/// enforced or ignored. +fn planning_run(workload: PlanningWorkload, ignore_accuracy: bool) -> PlanningRun { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap(); let capabilities = DeploymentCapabilities { @@ -63,14 +86,31 @@ fn selection( .with_deployment_capabilities(&capabilities) .with_ignore_accuracy_guarantees(ignore_accuracy); let roots = lower_promql_roots(&workload).unwrap(); - let run = run_planning_stages_over(roots, &entries, &data, models).unwrap(); - let (plan, selection) = run.run.selection.as_ref().expect("a plan is selected"); - let accepted = selection - .accepted_accuracy_per_query + run_planning_stages_over(roots, &entries, &data, models) + .unwrap() + .run +} + +/// The failure probability each readout the selected plan reads was +/// required to meet (`CandidateForMemoGroup::required_accuracy`), in node +/// order. +fn readout_failure_probabilities(run: &PlanningRun) -> Vec { + let (plan, _) = run.selection.as_ref().expect("a plan is selected"); + let physical = run.physical_asap_candidates.as_ref().expect("Stage 2 ran"); + plan.nodes .iter() - .map(ToString::to_string) - .collect(); - (plan.clone(), accepted) + .filter(|node| matches!(node.payload, Operator::ASAP(ASAPOp::SummaryEstimate { .. }))) + .filter_map(|node| { + let candidate = physical.physical_candidates[node.selected_physical_candidate?.0] + .logical_candidate_id; + let required = physical + .logical_candidates + .memo + .candidate(candidate) + .required_accuracy()?; + Some(required.failure_probability) + }) + .collect() } /// Queries A and B of the module's example. @@ -221,3 +261,92 @@ fn a_ratio_sharing_its_readouts_with_its_operands_still_meets_its_target() { "the ratio reads readouts composing within 0.05: {alphas:?}" ); } + +/// The p90 of `latency{side="a"}` or of `latency{side="b"}`, over the +/// whole result at relative 0.05 with δ = 0.01. Stage 0 bounds the result +/// by the series both branches read, 2,000, and the readouts read 2,000 +/// estimates, one per output value: every estimate read bounds it, so +/// Pass 2 gives each readout 0.01 / 2,000 = 5e-6 and (K) accepts them +/// (2,000 × 5e-6 = 0.01). Adding both sides' δ per output value against +/// 0.01 / 2,000 left this query no sketch plan. +#[test] +fn an_or_of_two_quantiles_reads_sketches() { + let run = planning_run( + workload( + &[( + r#"quantile_over_time(0.9, latency{side="a"}[5m]) or quantile_over_time(0.9, latency{side="b"}[5m])"#, + r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01}"#, + )], + LATENCY_SERIES, + ), + false, + ); + let (_, selection) = run.selection.as_ref().expect("a plan is selected"); + assert_eq!( + selection.accepted_accuracy_per_query[0].to_string(), + "accepted relative (DDSketch)" + ); + let levels = readout_failure_probabilities(&run); + assert!(!levels.is_empty()); + for level in levels { + assert!((level - 5e-6).abs() <= 1e-12, "{level}"); + } +} + +/// `max_over_time(latency[5m]) * on() group_left +/// quantile_over_time(0.99, reference[5m])`, `reference` one series +/// matched onto each of `latency`'s 1,000 (read exactly), over the +/// whole result at relative 0.01 with δ = 0.01: every output value reads +/// the one estimate, so the readout keeps δ = 0.01 and (K) accepts it. +/// Splitting δ over the 1,000 output values left it no sketch plan. +#[test] +fn a_quantile_matched_onto_every_series_reads_a_sketch() { + let run = planning_run( + workload_with_sources( + &[( + "max_over_time(latency[5m]) * on() group_left quantile_over_time(0.99, reference[5m])", + r#"{"accuracy_metric": "relative", "max_error": 0.02, "failure_probability": 0.01}"#, + )], + LATENCY_SERIES, + r#", "reference": {"series_count": 1, "metric_type": "gauge", "sample_interval": 1000, + "column_signs": {"value": "non_negative"}}"#, + ), + false, + ); + let (_, selection) = run.selection.as_ref().expect("a plan is selected"); + assert_eq!( + selection.accepted_accuracy_per_query[0].to_string(), + "accepted relative (DDSketch)" + ); + assert_eq!(readout_failure_probabilities(&run), [0.01]); +} + +/// The p90 of `latency` per value at δ = 0.01 (its own level 0.01) and +/// `max` of it over the whole result at δ = 0.01 (its own level 1e-5): +/// `max` reads all 1,000 estimates, so the readout it reads must fail +/// with at most 1e-5, whatever level the per-value query was given. +#[test] +fn a_max_reads_the_level_its_own_split_gives() { + let run = planning_run( + workload( + &[ + ( + "quantile_over_time(0.9, latency[5m])", + r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01, "accuracy_scope": "per_value"}"#, + ), + ( + "max(quantile_over_time(0.9, latency[5m]))", + r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01}"#, + ), + ], + LATENCY_SERIES, + ), + false, + ); + let levels = readout_failure_probabilities(&run); + assert!(!levels.is_empty()); + assert!( + levels.iter().all(|&level| level <= 1e-5 * (1.0 + 1e-9)), + "{levels:?}" + ); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/lib.rs b/crates/stage1-pass2-reuse-candidates/src/lib.rs index 618059393..3b6f942f7 100644 --- a/crates/stage1-pass2-reuse-candidates/src/lib.rs +++ b/crates/stage1-pass2-reuse-candidates/src/lib.rs @@ -70,6 +70,10 @@ pub enum Pass2RuleError { /// groups each query may read only exactly /// (`ErrorMetricsOfMemo::exact_only_groups_of_query`). /// +/// Step 3 also records the readout values each query's δ is split over, +/// whose bound Stage 3 checks +/// (`LogicalASAPMemoCandidates::readout_values_of_query`). +/// /// `entries[i]` is query `i`'s workload entry (its evaluation grid). pub fn generate_logical_asap_memo_candidates( pass1: Memo, @@ -151,5 +155,6 @@ pub fn generate_logical_asap_memo_candidates( pruned, evaluation_grid_of_group, exact_only_groups_of_query, + readout_values_of_query: readout_requirements.readout_values_of_query, }) } diff --git a/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs index 256ffbb5e..11a7c9df5 100644 --- a/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs +++ b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs @@ -7,6 +7,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::Pass2RuleError; use asap_accuracy::{readout_levels, AccuracyEvidence, ErrorMetricsOfMemo}; +use asap_types::accuracy::ReadoutValueCounts; use asap_types::stages::{ CandidateForMemoGroupId, LogicalRequirements, Memo, MemoGroupId, RequiredAccuracy, }; @@ -23,6 +24,9 @@ pub(crate) struct ReadoutRequirementsOutcome { /// failure probability, which it may read only exactly (another /// query's level says nothing of its own δ). pub(crate) exact_only_readout_groups_of_query: BTreeMap>, + /// Per query, by index: the readout values its δ is split over + /// (`ReadoutLevels::readout_values_of_query`). + pub(crate) readout_values_of_query: Vec>, } /// Gives every sketch readout (`SummaryEstimate` over a sketch state) its @@ -37,12 +41,14 @@ pub(crate) fn set_readout_requirements( order: &[MemoGroupId], evidence: &dyn AccuracyEvidence, ) -> Result { + let error_metrics = ErrorMetricsOfMemo::of(memo, order); + let levels = readout_levels(memo, &error_metrics, order, evidence); let mut outcome = ReadoutRequirementsOutcome { readouts_without_per_value_failure_probability: BTreeMap::new(), exact_only_readout_groups_of_query: BTreeMap::new(), + readout_values_of_query: levels.readout_values_of_query, }; - let error_metrics = ErrorMetricsOfMemo::of(memo, order); - for (readout, required) in readout_levels(memo, &error_metrics, order, evidence) { + for (readout, required) in levels.of_readout { for &query in &required.queries_without_per_value_failure_probability { outcome .exact_only_readout_groups_of_query diff --git a/crates/stage2-physical-optimization/src/test_fixtures.rs b/crates/stage2-physical-optimization/src/test_fixtures.rs index feb34a87a..edd9756e0 100644 --- a/crates/stage2-physical-optimization/src/test_fixtures.rs +++ b/crates/stage2-physical-optimization/src/test_fixtures.rs @@ -93,6 +93,7 @@ pub fn logical( ) -> LogicalASAPMemoCandidates { LogicalASAPMemoCandidates { exact_only_groups_of_query: vec![BTreeSet::new(); memo.queries().len()], + readout_values_of_query: vec![None; memo.queries().len()], memo, edges, failing_logical_edges: vec![], diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs index 15dd5564e..8033808a3 100644 --- a/crates/stage3-plan-selection/src/cost/tests.rs +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -152,6 +152,7 @@ fn no_execution_reading_a_producer_costs_less_than_nothing() { pruned: vec![], evaluation_grid_of_group: vec![None; groups], exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, candidates, edges, diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index d936a81b8..47d3579b2 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -128,10 +128,10 @@ pub fn select_plan( SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_per_output_value, + failure_probability_bound, } => ErrorSelectingPlan::AccuracyTargetBelowSolverPrecision { query: query.0, - failure_probability_per_output_value, + failure_probability_bound, }, })?; let plan = plan_of(physical, &solved, &outputs).map_err(|error| { diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index 5f530e122..48c808032 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -14,8 +14,11 @@ use std::hash::{Hash, Hasher}; use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; -use asap_accuracy::{query_roots_with_target, tolerated_limit, ErrorContract, ErrorMetricsOfMemo}; -use asap_types::accuracy::AccuracyMetric; +use asap_accuracy::{ + inputs_read_value_by_value, query_roots_with_target, ranks_by_estimate, + takes_each_value_from_one_input, tolerated_limit, ErrorContract, ErrorMetricsOfMemo, +}; +use asap_types::accuracy::{AccuracyMetric, FailureProbabilityBound}; use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId}; use super::equivalent_contexts::SharedVariables; @@ -23,20 +26,54 @@ use super::model::SelectionInputs; use crate::selectable::QueryId; /// An execution's error as (K) composes it: its error bound per metric -/// (by [`AccuracyMetric::index`]) and its failure probability. +/// (by [`AccuracyMetric::index`]) and two failure probabilities, each the +/// summed δ of estimates (one readout value in one evaluation) by the +/// union bound (`accuracy-models.md`, "Which estimates a result depends +/// on"). A failure probability is `None` when no declared fact bounds it +/// (a readout whose estimates per evaluation are unknown). /// -/// Example: a DDSketch readout required at relative 0.01 with δ = 1e-3 -/// is 0.01 in `Relative`, 0 in every other metric, and 1e-3. -#[derive(Debug, Clone, Copy, Default, PartialEq)] +/// Example: a DDSketch readout over 1000 series required at relative 0.01 +/// with δ = 1e-3 is 0.01 in `Relative`, 0 in every other metric, 1e-3 for +/// one value and 1000 × 1e-3 = 1 for all of them. +#[derive(Debug, Clone, Copy, PartialEq)] pub(super) struct ExecutionError { /// $E_{c,m}$ per metric. error_bound_per_metric: [f64; AccuracyMetric::COUNT], - /// $D_c$. - failure_probability: f64, + /// $P_c$: the largest, over its output values, of the summed δ of the + /// estimates one value depends on. + failure_probability_of_one_value: Option, + /// $W_c$: the summed δ of every estimate any of its output values + /// depends on. + failure_probability_of_all_values: Option, +} + +impl Default for ExecutionError { + /// No error, and no estimate to fail: an exact execution's. + fn default() -> Self { + Self { + error_bound_per_metric: [0.0; AccuracyMetric::COUNT], + failure_probability_of_one_value: Some(0.0), + failure_probability_of_all_values: Some(0.0), + } + } +} + +/// Sums of failure probabilities, `None` (unbounded) when any term is. +fn sum_of(terms: impl IntoIterator>) -> Option { + terms + .into_iter() + .try_fold(0.0, |total, term| Some(total + term?)) +} + +/// The largest of failure probabilities, `None` (unbounded) when any is. +fn largest_of(terms: impl IntoIterator>) -> Option { + terms + .into_iter() + .try_fold(0.0, |largest: f64, term| Some(largest.max(term?))) } impl ExecutionError { - /// The larger of `self` and `other` in each metric and in failure + /// The larger of `self` and `other` in each metric and in each failure /// probability, as the most any one of several producers passes on. fn largest(self, other: Self) -> Self { let mut largest = self; @@ -47,13 +84,75 @@ impl ExecutionError { { *error = error.max(other); } - largest.failure_probability = largest.failure_probability.max(other.failure_probability); + largest.failure_probability_of_one_value = largest_of([ + self.failure_probability_of_one_value, + other.failure_probability_of_one_value, + ]); + largest.failure_probability_of_all_values = largest_of([ + self.failure_probability_of_all_values, + other.failure_probability_of_all_values, + ]); largest } - /// Whether it is 0 in every metric and in failure probability. + /// Whether it is 0 in every metric and in each failure probability. fn is_zero(&self) -> bool { - self.failure_probability == 0.0 && self.error_bound_per_metric.iter().all(|&e| e == 0.0) + self.failure_probability_of_one_value == Some(0.0) + && self.failure_probability_of_all_values == Some(0.0) + && self.error_bound_per_metric.iter().all(|&e| e == 0.0) + } + + /// The failure probability `bound` checks: of all values for every + /// estimate read, of one value for each covered value. + fn failure_probability_checked_by(&self, bound: &FailureProbabilityBound) -> Option { + match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => { + self.failure_probability_of_all_values + } + FailureProbabilityBound::EachCoveredValue { .. } => { + self.failure_probability_of_one_value + } + } + } +} + +/// How an execution's output values read its inputs, which decides how +/// (K) composes its error and failure probabilities (`asap_accuracy`'s +/// operator classes, the ones Pass 2's split counts by). +#[derive(Debug, Clone)] +struct HowInputsAreRead { + /// Its first so many inputs are read value by value: each output value + /// reads one value of each ([`inputs_read_value_by_value`]); every + /// other input is read whole. `None`: all are read whole (a reduction). + value_by_value: Option, + /// Each output value is copied from one input (`or`, a concatenation: + /// [`takes_each_value_from_one_input`]), so its error is the largest of + /// the inputs', not their sum. + each_value_from_one_input: bool, + /// Each output value depends on the estimates of one input only + /// (`ErrorMetricsOfMemo::each_value_depends_on_one_input`): the inputs + /// read value by value add the largest of their one-value failure + /// probabilities, not their sum. Not for `topk(3, q) or r`, whose + /// values from `r` depend on the top-k's choice too. + each_value_depends_on_one_input: bool, + /// Per input: whether an estimate it depends on can make the output + /// miss (`ErrorMetricsOfMemo::input_estimates_reach_output`), so its + /// failure probabilities add to the output's. Not a count's input, nor + /// the right side of `and` and `unless`, unless a top-k chooses its + /// values. + estimates_reach_output: Vec, +} + +impl HowInputsAreRead { + /// Whether input `input` is read value by value. + fn reads_value_by_value(&self, input: usize) -> bool { + self.value_by_value.is_some_and(|count| input < count) + } + + /// Whether an estimate input `input` depends on can make the output + /// miss ([`Self::estimates_reach_output`]). + fn estimates_of_input_reach_output(&self, input: usize) -> bool { + self.estimates_reach_output[input] } } @@ -61,7 +160,8 @@ impl ExecutionError { /// ([`AccuracyRules::compose`]). struct Composition { /// Its own error plus each input's error times the contract's factor, - /// and its own failure probability plus each input's. + /// and its own failure probabilities plus its inputs' by the operator + /// classes. error: ExecutionError, /// Whether every input's error passes the execution's metric rule (it /// has a contract in that metric) and the contract's validity cap. @@ -80,8 +180,13 @@ pub(super) struct AccuracyRules { /// its logical candidate. candidate_of_execution: Vec, /// Per execution, the error it adds: a readout's requirement - /// (`CandidateForMemoGroup::required_accuracy`), 0 for any other. + /// (`CandidateForMemoGroup::required_accuracy`), its δ for one value + /// (`R` times it for a top-k, which ranks every estimate) and `R` + /// times its δ for all values (`Memo::max_estimates_per_evaluation`; + /// 0 at δ = 0 whatever `R`); 0 for any other. own_error_per_execution: Vec, + /// Per execution, how its output values read its inputs. + how_inputs_are_read: Vec, /// Per execution, whether a query with an accuracy target reaches its /// group. reached_by_query_with_target: Vec, @@ -94,34 +199,42 @@ pub(super) struct AccuracyRules { roots: HashMap<(QueryId, MemoGroupId), RootLimits>, } -/// What a root check of (K) bounds one query output by, each limit -/// already widened by [`tolerated_limit`] as the edge check widens it. -/// Two outputs with equal limits have the same root rows, so they compare -/// and hash by the limits' bits. +/// What a root check of (K) bounds one query output by. The limits are +/// widened by [`tolerated_limit`] where they are compared, as the edge +/// check widens them. Two outputs with equal limits have the same root +/// rows, so they compare and hash by the limits' bits. /// /// Example: `p99 / p50` at relative 0.03 with δ = 0.01 over a whole result -/// of 100 values allows 0.03 (widened) in `Relative`, 0 in every other -/// metric, and 1e-4 (widened) failure probability per output value. +/// of 100 values, reading two readouts over 100 series each, allows 0.03 +/// in `Relative`, 0 in every other metric, and checks each covered value: +/// its one-value failure probability at most 1e-4. #[derive(Debug, Clone, Copy)] pub(super) struct RootLimits { /// Per metric (by [`AccuracyMetric::index`]), the error the output may /// have: the target's ε in a metric the root allows /// ([`query_roots_with_target`]), 0 in any other. tolerated_error_per_metric: [f64; AccuracyMetric::COUNT], - /// The failure probability each output value may have - /// (`AccuracyTarget::failure_probability_per_output_value`); `None` - /// without a δ, or with one over the whole result and $n_q$ unknown - /// (Pass 2 then gives that query's readouts no level). - tolerated_failure_probability_per_output_value: Option, + /// The union bound the query's failure probability is checked by, with + /// its limit (the bound Pass 2 split δ by, + /// `LogicalASAPMemoCandidates::failure_probability_bound`); + /// `None` without a δ, or with neither bound known (Pass 2 then gives + /// the query's readouts no level, and (X) makes it read them exactly). + failure_probability_bound: Option, } impl RootLimits { - /// The limits' bits, which equality, hashing and order compare. - fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option) { + /// The limits' bits, which equality, hashing and order compare: the + /// error per metric, and the bound's kind and limit (the rows read no + /// more of it). + fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option<(bool, u64)>) { ( self.tolerated_error_per_metric.map(f64::to_bits), - self.tolerated_failure_probability_per_output_value - .map(f64::to_bits), + self.failure_probability_bound.map(|bound| { + ( + matches!(bound, FailureProbabilityBound::EachCoveredValue { .. }), + bound.limit().to_bits(), + ) + }), ) } } @@ -181,13 +294,44 @@ impl AccuracyRules { .iter() .map(|&id| { let mut own = ExecutionError::default(); - if let Some(required) = memo.candidate(id).required_accuracy() { + let candidate = memo.candidate(id); + if let Some(required) = candidate.required_accuracy() { own.error_bound_per_metric[required.metric.index()] = required.max_error; - own.failure_probability = required.failure_probability; + let delta = required.failure_probability; + // At δ = 0 no estimate can fail, however many there are. + let of_every_estimate = if delta == 0.0 { + Some(0.0) + } else { + memo.max_estimates_per_evaluation(id.group) + .map(|estimates| estimates as f64 * delta) + }; + own.failure_probability_of_one_value = + if ranks_by_estimate(&candidate.expression.operator) { + of_every_estimate + } else { + Some(delta) + }; + own.failure_probability_of_all_values = of_every_estimate; } own }) .collect(); + let error_metrics = ErrorMetricsOfMemo::of(memo, &order); + let how_inputs_are_read = candidate_of_execution + .iter() + .map(|&id| { + let operator = &memo.candidate(id).expression.operator; + HowInputsAreRead { + value_by_value: inputs_read_value_by_value(operator), + each_value_from_one_input: takes_each_value_from_one_input(operator), + each_value_depends_on_one_input: error_metrics + .each_value_depends_on_one_input(memo, id), + estimates_reach_output: (0..operator.children().len()) + .map(|input| error_metrics.input_estimates_reach_output(memo, id, input)) + .collect(), + } + }) + .collect(); let reached_by_query_with_target = candidate_of_execution .iter() .map(|id| { @@ -208,25 +352,24 @@ impl AccuracyRules { let roots = query_roots_with_target(memo) .into_iter() .map(|(query, root, target, allowed_metrics)| { - let failure_probability = target.failure_probability_per_output_value( - memo.queries()[query].number_of_values_per_query_evaluation, - ); + let failure_probability_bound = + physical.logical_candidates.failure_probability_bound(query); let mut tolerated_error_per_metric = [0.0; AccuracyMetric::COUNT]; for metric in allowed_metrics { tolerated_error_per_metric[metric.index()] = tolerated_limit(target.max_error); } let limits = RootLimits { tolerated_error_per_metric, - tolerated_failure_probability_per_output_value: failure_probability - .map(tolerated_limit), + failure_probability_bound, }; ((QueryId(query), root), limits) }) .collect(); Some(Self { - error_metrics: ErrorMetricsOfMemo::of(memo, &order), + error_metrics, candidate_of_execution, own_error_per_execution, + how_inputs_are_read, reached_by_query_with_target, executions_producers_first, roots, @@ -253,28 +396,53 @@ impl AccuracyRules { /// (`accuracy-models.md`, "Checking each query's target in /// selection"): its own error /// plus, per metric, $\beta_{c,i,m}$ times each input's error in that - /// metric where the input's error reaches the output ($\beta > 0$), - /// and its own failure probability plus every input's (a union bound). + /// metric where the input's error reaches the output ($\beta > 0$); the + /// largest of those instead when each output value is copied from one + /// input (`or`, a concatenation), whose error is that input's. Its + /// failure probabilities add, over the inputs whose error reaches + /// the output in some metric (a union bound over estimates): + /// - of one value, its own plus the one-value failure probability of + /// each input read value by value (the largest of them when each + /// value is copied from one input) plus the all-values one of each + /// input read whole; + /// - of all values, its own plus each input's all-values one. + /// /// An input's error in a metric `c` has no contract for breaks the /// metric rule, and one above the contract's validity cap $b_{c,i}$ /// breaks the cap: either leaves the inputs not admitted, and adds /// nothing in that metric for the metric rule. /// - /// Example: `p99 / p50`'s contract passes each operand's relative + /// Examples: `p99 / p50`'s contract passes each operand's relative /// error with factor 1.111 and caps the denominator at 0.1: reading /// 0.02 and 0.02 composes 0.044, admitted; reading a denominator at - /// 0.2 is not admitted. + /// 0.2 is not admitted. `max` over a readout of 1000 series at δ 1e-5 + /// reads it whole: 0.01 for one value and for all. `q or r` over + /// readouts at δ 1e-4 of 100 series each: 1e-4 for one value, 0.02 + /// for all. fn compose( &self, c: usize, inputs_read: impl IntoIterator, ) -> Composition { + let own = self.own_error_per_execution[c]; let mut composed = Composition { - error: self.own_error_per_execution[c], + error: own, inputs_admitted: true, }; + let how = &self.how_inputs_are_read[c]; + let mut one_value_of_inputs_read_value_by_value = Vec::new(); + let mut all_values_of_inputs_read_whole = Vec::new(); + let mut all_values_of_inputs = Vec::new(); for (i, read) in inputs_read { - composed.error.failure_probability += read.failure_probability; + if how.estimates_of_input_reach_output(i) { + all_values_of_inputs.push(read.failure_probability_of_all_values); + if how.reads_value_by_value(i) { + one_value_of_inputs_read_value_by_value + .push(read.failure_probability_of_one_value); + } else { + all_values_of_inputs_read_whole.push(read.failure_probability_of_all_values); + } + } for (m, &error) in read.error_bound_per_metric.iter().enumerate() { if error <= 0.0 { continue; @@ -291,10 +459,30 @@ impl AccuracyRules { { composed.inputs_admitted = false; } - composed.error.error_bound_per_metric[m] += - contract.error_factor_per_input[i] * error; + let of_input = contract.error_factor_per_input[i] * error; + let composed_error = &mut composed.error.error_bound_per_metric[m]; + if how.each_value_from_one_input { + // Each output value's error is one input's. + *composed_error = composed_error.max(own.error_bound_per_metric[m] + of_input); + } else { + *composed_error += of_input; + } } } + let of_inputs_read_value_by_value = if how.each_value_depends_on_one_input { + largest_of(one_value_of_inputs_read_value_by_value) + } else { + sum_of(one_value_of_inputs_read_value_by_value) + }; + composed.error.failure_probability_of_one_value = sum_of([ + own.failure_probability_of_one_value, + of_inputs_read_value_by_value, + sum_of(all_values_of_inputs_read_whole), + ]); + composed.error.failure_probability_of_all_values = sum_of([ + own.failure_probability_of_all_values, + sum_of(all_values_of_inputs), + ]); composed } @@ -304,8 +492,11 @@ impl AccuracyRules { /// meets (K), exactly in f64: each execution a query with a target /// reaches, composed ([`Self::compose`]) from the producers selected /// on its inputs, admits its inputs' errors and, at each output of a - /// query with a target, is within the root's limits. `Err` names the - /// first query (in `answering` order) whose output misses its target. + /// query with a target, is within the root's error limits; and each + /// query's failure probability its bound checks, summed over its + /// outputs (a query with several roots reads them all), is within its + /// limit. `Err` names the first query (in `answering` order) whose + /// output misses its target. /// /// Example: p99 / p50 at relative 0.03 over two readouts at 0.02 does /// not meet it (1.111 · 0.04 = 0.044), so it is `Err` with the ratio's @@ -327,6 +518,11 @@ impl AccuracyRules { let mut composed: Vec> = vec![None; count]; let mut on_walk = vec![false; count]; let memo = &inputs.physical.logical_candidates.memo; + // Per query with a δ, in `answering` order: its bound and the + // failure probability it checks, summed over the outputs so far + // (`None` once one is unbounded). + let mut failure_probability_of_query: Vec<(QueryId, FailureProbabilityBound, Option)> = + Vec::new(); for (&(_, group), &(query, v)) in memo.output_groups().iter().zip(answering) { // Compose `v` and every execution below it, producers first. let mut pending: Vec<(usize, bool)> = vec![(v, false)]; @@ -377,12 +573,24 @@ impl AccuracyRules { .iter() .zip(limits.tolerated_error_per_metric) .any(|(&error, limit)| error > limit) - || limits - .tolerated_failure_probability_per_output_value - .is_some_and(|limit| error.failure_probability > limit) { return Err(query); } + if let Some(bound) = limits.failure_probability_bound { + let checked = error.failure_probability_checked_by(&bound); + match failure_probability_of_query + .iter_mut() + .find(|(known, _, _)| *known == query) + { + Some((_, _, total)) => *total = sum_of([*total, checked]), + None => failure_probability_of_query.push((query, bound, checked)), + } + } + } + for (query, bound, total) in failure_probability_of_query { + if total.is_none_or(|total| total > tolerated_limit(bound.limit())) { + return Err(query); + } } Ok(()) } @@ -414,17 +622,25 @@ pub(super) struct ErrorModel<'a> { /// $y$: (position in `SelectableExecutions::edges`, producer /// execution); empty for an execution without rows. selectable_edges_per_input: Vec>>, - /// $M$: the largest error and failure probability any selection gives - /// the execution. + /// $M$ (and $M^P$, $M^W$): the largest error and failure probabilities + /// any selection gives the execution; a failure probability is `None` + /// when some selection leaves it unbounded. ceiling: Vec, + /// Whether some query root above the execution checks its failure + /// probability of one value ($P$) and of all values ($W$), through the + /// rows it reads them by: a root checking each covered value reads $P$, + /// one checking every estimate read $W$; $P$ reads $P$ of an input read + /// value by value and $W$ of one read whole, $W$ reads $W$. (K) has a + /// variable only for a failure probability some root checks. + checked_failure_probabilities: Vec<(bool, bool)>, /// Whether the execution's error is constant: its inputs carry none /// under any selection, so its ceilings are its own error. error_is_constant: Vec, } -/// An execution's error in one metric, or its failure probability, as -/// (K)'s rows read it: a constant (a readout's -/// own requirement) or a variable with its ceiling. +/// An execution's error in one metric, or one of its failure +/// probabilities, as (K)'s rows read it: a constant (a readout's own +/// requirement) or a variable with its ceiling. #[derive(Clone, Copy)] enum ErrorTerm { Constant(f64), @@ -468,9 +684,11 @@ impl<'a> ErrorModel<'a> { selectable_edges_per_input: (0..count).map(selectable_edges_of).collect(), has_rows, ceiling: vec![ExecutionError::default(); count], + checked_failure_probabilities: vec![(false, false); count], error_is_constant: vec![false; count], }; model.set_ceilings(); + model.set_checked_failure_probabilities(inputs, shared); model.error_is_constant = (0..count) .map(|c| { model.has_rows[c] @@ -497,17 +715,121 @@ impl<'a> ErrorModel<'a> { } } - /// Execution `u`'s failure probability; `None` when it is 0 - /// under every selection. - fn failure_probability(&self, accuracy: &AccuracyVariables, u: usize) -> Option { - let ceiling = self.ceiling[u].failure_probability; - match (ceiling > 0.0, self.error_is_constant[u]) { + /// Execution `u`'s failure probability of one value ($P_u$); `Ok(None)` + /// when it is 0 under every selection, `Err` when some selection + /// leaves it unbounded. + fn failure_probability_of_one_value( + &self, + accuracy: &AccuracyVariables, + u: usize, + ) -> Result, Unbounded> { + self.failure_probability_term( + self.ceiling[u].failure_probability_of_one_value, + u, + &accuracy.failure_probability_of_one_value, + ) + } + + /// Execution `u`'s failure probability of all values ($W_u$), as + /// [`Self::failure_probability_of_one_value`]. + fn failure_probability_of_all_values( + &self, + accuracy: &AccuracyVariables, + u: usize, + ) -> Result, Unbounded> { + self.failure_probability_term( + self.ceiling[u].failure_probability_of_all_values, + u, + &accuracy.failure_probability_of_all_values, + ) + } + + /// The failure probability `variables` holds for `u`, whose ceiling is + /// `ceiling`, as a term. + fn failure_probability_term( + &self, + ceiling: Option, + u: usize, + variables: &HashMap, + ) -> Result, Unbounded> { + let ceiling = ceiling.ok_or(Unbounded)?; + Ok(match (ceiling > 0.0, self.error_is_constant[u]) { (false, _) => None, (true, true) => Some(ErrorTerm::Constant(ceiling)), (true, false) => Some(ErrorTerm::Variable { - variable: accuracy.failure_probability[&u], + variable: variables[&u], ceiling, }), + }) + } + + /// The failure probability `bound` checks of execution `v` (of all + /// values for every estimate read, of one value for each covered + /// value). + fn failure_probability_checked_by( + &self, + accuracy: &AccuracyVariables, + v: usize, + bound: &FailureProbabilityBound, + ) -> Result, Unbounded> { + match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => { + self.failure_probability_of_all_values(accuracy, v) + } + FailureProbabilityBound::EachCoveredValue { .. } => { + self.failure_probability_of_one_value(accuracy, v) + } + } + } + + /// [`Self::checked_failure_probabilities`], in one pass from each + /// original query's answering executions down, consumers first. + fn set_checked_failure_probabilities( + &mut self, + inputs: &SelectionInputs<'_>, + shared: &SharedVariables, + ) { + let rules = self.rules; + let memo = &inputs.physical.logical_candidates.memo; + for (&(query, group), (query_id, executions)) in memo + .output_groups() + .iter() + .zip(inputs.selectable.answering()) + { + if shared.original_of_query[query] != *query_id { + continue; + } + let Some(bound) = rules + .root_limits(*query_id, group) + .and_then(|limits| limits.failure_probability_bound) + else { + continue; + }; + for &v in executions { + let checked = &mut self.checked_failure_probabilities[v]; + match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => checked.1 = true, + FailureProbabilityBound::EachCoveredValue { .. } => checked.0 = true, + } + } + } + for &c in rules.executions_producers_first.iter().rev() { + let (one_value, all_values) = self.checked_failure_probabilities[c]; + if !self.has_rows[c] || !(one_value || all_values) { + continue; + } + let how = &rules.how_inputs_are_read[c]; + for (i, edges) in self.selectable_edges_per_input[c].iter().enumerate() { + if !how.estimates_of_input_reach_output(i) { + continue; + } + let value_by_value = how.reads_value_by_value(i); + for &(_, u) in edges { + let checked = &mut self.checked_failure_probabilities[u]; + checked.0 |= one_value && value_by_value; + checked.1 |= all_values || (one_value && !value_by_value); + } + } } } @@ -549,24 +871,35 @@ impl<'a> ErrorModel<'a> { } } +/// A failure probability no declared fact bounds under some selection +/// (a readout whose values per evaluation are unknown, below): (K) has no +/// variable for it, and no output may be answered by a bound reading it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Unbounded; + /// The accuracy variables of constraint (K), all continuous, in /// each only for an execution with rows /// ([`ErrorModel`]) whose error is not constant and whose ceiling is above -/// 0, by execution position. +/// 0 (and, for a failure probability, bounded and checked by some query +/// root above: [`ErrorModel::checked_failure_probabilities`]), by +/// execution position. pub(super) struct AccuracyVariables { /// $E_{\kappa,c,m}$: the execution's error bound in metric $m$ (by /// [`AccuracyMetric::index`]). error_bound: HashMap<(usize, usize), Variable>, - /// $D_{\kappa,c}$: its failure probability. - failure_probability: HashMap, + /// $P_{\kappa,c}$: its failure probability of one value. + failure_probability_of_one_value: HashMap, + /// $W_{\kappa,c}$: its failure probability of all values. + failure_probability_of_all_values: HashMap, } impl AccuracyVariables { - /// $E$ and $D$ over `model`, added to `problem`. + /// $E$, $P$ and $W$ over `model`, added to `problem`. pub(super) fn new(model: &ErrorModel<'_>, problem: &mut ProblemVariables) -> Self { let mut variables = Self { error_bound: HashMap::new(), - failure_probability: HashMap::new(), + failure_probability_of_one_value: HashMap::new(), + failure_probability_of_all_values: HashMap::new(), }; for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) @@ -578,47 +911,74 @@ impl AccuracyVariables { .insert((c, m), problem.add(variable().min(0))); } } - if model.ceiling[c].failure_probability > 0.0 { - variables - .failure_probability - .insert(c, problem.add(variable().min(0))); + let ceiling = model.ceiling[c]; + let (one_value_checked, all_values_checked) = model.checked_failure_probabilities[c]; + for (failure_probability, checked, of) in [ + ( + ceiling.failure_probability_of_one_value, + one_value_checked, + &mut variables.failure_probability_of_one_value, + ), + ( + ceiling.failure_probability_of_all_values, + all_values_checked, + &mut variables.failure_probability_of_all_values, + ), + ] { + if checked && failure_probability.is_some_and(|ceiling| ceiling > 0.0) { + of.insert(c, problem.add(variable().min(0))); + } } } variables } - /// The number of $E$ and $D$ variables. + /// The number of $E$, $P$ and $W$ variables. #[cfg(test)] pub(super) fn variable_count(&self) -> usize { - self.error_bound.len() + self.failure_probability.len() + self.error_bound.len() + + self.failure_probability_of_one_value.len() + + self.failure_probability_of_all_values.len() } } /// (K) (`accuracy-models.md`, "Checking each query's target in /// selection"), per execution $c$ with rows ([`ErrorModel`]): /// - $E_{c,m} \ge \varepsilon_c[m = m_c] + \sum_i \beta_{c,i,m} -/// F_{c,i,m}$ and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per +/// F_{c,i,m}$ (one row per input instead of the sum when each output +/// value is copied from one input, `or` and a concatenation) and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per /// selectable edge, $F_{c,i,m} \le b_{c,i}$ where the contract's linear /// bound holds only up to $b_{c,i}$ (a ratio's denominator), and /// $E_{u,m} \le M_{u,m}(1 - y_{u,c,i})$ when $c$ has no contract in $m$ /// (the metric rule on the selected path); an input whose error does not /// reach the output ($\beta = 0$, a count's) passes none; -/// - $D_c \ge \delta_c + \sum_i H_{c,i}$, $H_{c,i} \ge D_u - M_u(1 - -/// y_{u,c,i})$ (a union bound); +/// - failure probabilities, over the inputs whose error reaches the output +/// in some metric, with $HP_{c,i} \ge P_u - M^P_u(1 - y_{u,c,i})$ and +/// $HW_{c,i} \ge W_u - M^W_u(1 - y_{u,c,i})$ (a union bound over +/// estimates, [`AccuracyRules::compose`]): +/// $P_c \ge P^{own}_c + \sum_{i\ \text{value by value}} HP_{c,i} + +/// \sum_{i\ \text{whole}} HW_{c,i}$, one row per input read value by +/// value instead of their sum when each value is copied from one input +/// (`or`), and $W_c \ge W^{own}_c + \sum_i HW_{c,i}$; /// - per query $q$ with a target, output group $g$ its root reads and /// execution $v$ answering it, $E_{v,m} \le \varepsilon_q[m \in /// \text{allowed}] + M(1 - z_{q,v})$, the metrics $g$ allows at the root /// ([`query_roots_with_target`]: the target's at an operator root, none -/// for the operands of a scalar expression no contract covers), and -/// $D_v \le \delta_q + M(1 - z_{q,v})$ with $\delta_q$ per output value -/// (`AccuracyTarget::failure_probability_per_output_value`); no $D$ row -/// without one ([`RootLimits`], each limit widened by -/// [`tolerated_limit`]). +/// for the operands of a scalar expression no contract covers); +/// - per query $q$ with a δ, one failure row by the bound +/// `AccuracyTarget::failure_probability_bound` names: $W_v \le \delta_q$ +/// (every estimate read) or $P_v \le \delta_q / V$ (each covered value) +/// $+ M(1 - z_{q,v})$; a query with several roots sums the term its +/// answering executions give per root, $T_{q,g} \ge X_v - M_v(1 - +/// z_{q,v})$, in one row $\sum_g T_{q,g} \le$ the limit. An output whose +/// term is unbounded under some selection is not answered through that +/// execution. No row without a bound ([`RootLimits`], each limit widened +/// by [`tolerated_limit`]). /// /// It is [`AccuracyRules::compose`]'s rule over variables: the same own /// error, factors, metric rule and validity caps. /// -/// $F$ and $H$ are added to `problem` here. A copy of an equivalent context +/// $F$, $HP$, $HW$ and $T$ are added to `problem` here. A copy of an equivalent context /// and its queries have no rows: they share their original's variables, /// and the copy's rows would be its original's /// ([`super::equivalent_contexts::copies_in_equivalent_contexts`] pairs @@ -636,9 +996,17 @@ pub(super) fn accuracy_constraints( let mut constraints = Vec::new(); for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) { - // Σ_i β_{c,i,m} F_{c,i,m} per metric, and Σ_i H_{c,i}. - let mut error_of_inputs: [Expression; AccuracyMetric::COUNT] = Default::default(); - let mut failure_probability_of_inputs = Expression::default(); + // β_{c,i,m} F_{c,i,m} per metric and input: summed, or each alone + // when every output value is copied from one input. + let mut error_of_inputs: [Vec; AccuracyMetric::COUNT] = Default::default(); + // HP_{c,i} of each input read value by value, Σ HW_{c,i} of the + // inputs read whole, and Σ_i HW_{c,i}. + let has_one_value = accuracy.failure_probability_of_one_value.contains_key(&c); + let has_all_values = accuracy.failure_probability_of_all_values.contains_key(&c); + let how = &rules.how_inputs_are_read[c]; + let mut one_value_of_inputs_read_value_by_value: Vec = Vec::new(); + let mut all_values_of_inputs_read_whole = Expression::default(); + let mut all_values_of_inputs = Expression::default(); for (i, edges) in model.selectable_edges_per_input[c].iter().enumerate() { for (m, error_of_input) in error_of_inputs.iter_mut().enumerate() { let read: Vec<(Variable, ErrorTerm)> = edges @@ -670,46 +1038,102 @@ pub(super) fn accuracy_constraints( let holds_up_to = tolerated_limit(holds_up_to); constraints.push(constraint!(of_input.clone() <= holds_up_to)); } - *error_of_input += contract.error_factor_per_input[i] * of_input; + error_of_input.push(contract.error_factor_per_input[i] * of_input); } - // H_{c,i} counts the failure of the one producer selected on - // input i, not of each readout value an output value depends - // on: `max(q)` over 1,000 series reads 1,000 readout values, - // and D here adds the readout's δ once. That is sound only - // because Pass 2's levels already split the query's δ over - // the readout values its output depends on - // (`readout_caps::query_levels`), so each readout's δ is the - // query's over 1,000; the root row checks the composed sum - // against the per-output-value δ. - let read: Vec<(Variable, ErrorTerm)> = edges - .iter() - .filter_map(|&(position, u)| { - Some((y[&position], model.failure_probability(accuracy, u)?)) - }) - .collect(); - if !read.is_empty() { - failure_probability_of_inputs += chosen_producers(&read, &mut constraints, problem); + // Only an input whose estimates reach the output can make it + // miss: a count's input and the right of `and` add nothing, + // unless a top-k chooses their values. + if !how.estimates_of_input_reach_output(i) { + continue; + } + let value_by_value = how.reads_value_by_value(i); + // A variable's ceiling is bounded only when every producer's + // is, so each term read here is bounded. + let read_of = |of_all_values: bool| { + edges + .iter() + .filter_map(|&(position, u)| { + let term = if of_all_values { + model.failure_probability_of_all_values(accuracy, u) + } else { + model.failure_probability_of_one_value(accuracy, u) + }; + let term = term.expect("a bounded ceiling has bounded producers")?; + Some((y[&position], term)) + }) + .collect::>() + }; + if has_one_value && value_by_value { + let read = read_of(false); + if !read.is_empty() { + one_value_of_inputs_read_value_by_value.push(chosen_producers( + &read, + &mut constraints, + problem, + )); + } + } + if has_all_values || (has_one_value && !value_by_value) { + let read = read_of(true); + if !read.is_empty() { + let of_input = chosen_producers(&read, &mut constraints, problem); + if !value_by_value { + all_values_of_inputs_read_whole += of_input.clone(); + } + all_values_of_inputs += of_input; + } } } let own = rules.own_error_per_execution[c]; for (m, of_inputs) in error_of_inputs.into_iter().enumerate() { if let Some(&error_bound) = accuracy.error_bound.get(&(c, m)) { let own = own.error_bound_per_metric[m]; - constraints.push(constraint!(error_bound >= own + of_inputs)); + if how.each_value_from_one_input && !of_inputs.is_empty() { + for of_input in of_inputs { + constraints.push(constraint!(error_bound >= own + of_input)); + } + } else { + let of_inputs: Expression = of_inputs.into_iter().sum(); + constraints.push(constraint!(error_bound >= own + of_inputs)); + } } } - if let Some(&failure_probability) = accuracy.failure_probability.get(&c) { - let own = own.failure_probability; - constraints.push(constraint!( - failure_probability >= own + failure_probability_of_inputs - )); + if let Some(&of_one_value) = accuracy.failure_probability_of_one_value.get(&c) { + let own = own + .failure_probability_of_one_value + .expect("a bounded ceiling has a bounded own term"); + let whole = all_values_of_inputs_read_whole; + if how.each_value_depends_on_one_input + && !one_value_of_inputs_read_value_by_value.is_empty() + { + for of_input in one_value_of_inputs_read_value_by_value { + constraints.push(constraint!(of_one_value >= own + of_input + whole.clone())); + } + } else { + let value_by_value: Expression = + one_value_of_inputs_read_value_by_value.into_iter().sum(); + constraints.push(constraint!(of_one_value >= own + value_by_value + whole)); + } + } + if let Some(&of_all_values) = accuracy.failure_probability_of_all_values.get(&c) { + let own = own + .failure_probability_of_all_values + .expect("a bounded ceiling has a bounded own term"); + constraints.push(constraint!(of_all_values >= own + all_values_of_inputs)); } } let memo = &inputs.physical.logical_candidates.memo; - for (&(query, group), (query_id, executions)) in memo - .output_groups() - .iter() - .zip(inputs.selectable.answering()) + let output_groups = memo.output_groups(); + let mut outputs_per_query = vec![0usize; memo.queries().len()]; + for &(query, _) in &output_groups { + outputs_per_query[query] += 1; + } + // Per query with a bound and several outputs: the bound and, per + // output, what its answering execution gives. + let mut several_roots: BTreeMap = + BTreeMap::new(); + for (&(query, group), (query_id, executions)) in + output_groups.iter().zip(inputs.selectable.answering()) { debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); if shared.original_of_query[query] != *query_id { @@ -718,6 +1142,8 @@ pub(super) fn accuracy_constraints( let Some(limits) = rules.root_limits(*query_id, group) else { continue; }; + let outputs_of_query = outputs_per_query[query]; + let mut checked_per_execution: Vec<(Variable, ErrorTerm)> = Vec::new(); for &v in executions { let z = z[&(*query_id, v)]; for (m, &limit) in limits.tolerated_error_per_metric.iter().enumerate() { @@ -727,13 +1153,36 @@ pub(super) fn accuracy_constraints( .and_then(|error| at_most_when_answering(error, limit, z)), ); } - if let (Some(limit), Some(failure_probability)) = ( - limits.tolerated_failure_probability_per_output_value, - model.failure_probability(accuracy, v), - ) { - constraints.extend(at_most_when_answering(failure_probability, limit, z)); + let Some(bound) = limits.failure_probability_bound else { + continue; + }; + match model.failure_probability_checked_by(accuracy, v, &bound) { + Err(Unbounded) => constraints.push(constraint!(z <= 0)), + Ok(None) => {} + Ok(Some(term)) if outputs_of_query == 1 => { + constraints.extend(at_most_when_answering( + term, + tolerated_limit(bound.limit()), + z, + )); + } + Ok(Some(term)) => checked_per_execution.push((z, term)), } } + if let (Some(bound), true) = (limits.failure_probability_bound, outputs_of_query > 1) { + let of_output = if checked_per_execution.is_empty() { + Expression::default() + } else { + chosen_producers(&checked_per_execution, &mut constraints, problem) + }; + let (_, total) = several_roots + .entry(*query_id) + .or_insert_with(|| (bound, Expression::default())); + *total += of_output; + } + } + for (bound, total) in several_roots.into_values() { + constraints.push(constraint!(total <= tolerated_limit(bound.limit()))); } constraints } diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 86515e042..ebf47518b 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -22,6 +22,7 @@ use good_lp::{ constraint, variable, Constraint, Expression, ProblemVariables, SolverModel, Variable, }; +use asap_types::accuracy::FailureProbabilityBound; use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; @@ -391,13 +392,15 @@ pub(crate) enum SelectionMilpError { Solver(String), /// The solver's selection misses `query`'s accuracy target when /// checked exactly: its limits, such as a failure probability of - /// 1e-12 per output value (δ = 0.01 over 1e10 values), are below the + /// 1e-12 for each covered value (δ = 0.01 over 1e10 values), are below the /// solver's least feasibility tolerance, /// [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`], so (K) cannot be trusted. TargetBelowSolverPrecision { query: QueryId, - /// The target's δ per output value; `None` without a δ. - failure_probability_per_output_value: Option, + /// The union bound the target's δ is checked by, with its limit + /// (`AccuracyTarget::failure_probability_bound`); `None` without + /// one. + failure_probability_bound: Option, }, } @@ -556,19 +559,12 @@ pub(crate) fn solve_selection( // The query the first solve's selection missed, checked exactly: the // error names it if the second solve finds no plan at all. let mut missed_at_default_tolerance: Option = None; - let below_solver_precision = |query: QueryId| { - let memo = &inputs.physical.logical_candidates.memo; - let memo_query = &memo.queries()[query.0]; - SelectionMilpError::TargetBelowSolverPrecision { - query, - failure_probability_per_output_value: memo_query.accuracy_target.as_ref().and_then( - |target| { - target.failure_probability_per_output_value( - memo_query.number_of_values_per_query_evaluation, - ) - }, - ), - } + let below_solver_precision = |query: QueryId| SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_bound: inputs + .physical + .logical_candidates + .failure_probability_bound(query.0), }; let (selected, selected_edges, answering) = loop { let started = Instant::now(); diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 59dfdc616..50fe3f8fd 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -4,15 +4,21 @@ use std::collections::{BTreeSet, HashMap}; -use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::operator::Reduction; +use asap_types::accuracy::{ + AccuracyMetric, AccuracyScope, AccuracyTarget, FailureProbabilityBound, +}; +use asap_types::ir::operator::arguments::{ + BinaryOpKind, GroupSide, PromQLVectorSetOpKind, VectorGrouping, VectorMatch, VectorMatchKind, +}; +use asap_types::ir::operator::{AggIntent, Reduction}; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::expr_ir::CompareOpKind; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::scalar::ExprSemantics; use asap_types::ir::scalar::ScalarValue; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, Operator}; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; use asap_types::ir::{QueryRoot, ScalarExpr}; use asap_types::stages::{ CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalRequirements, @@ -25,11 +31,11 @@ use asap_types::summary_type::{ }; use asap_types::test_support::memo_builders::{ group, own_candidate, quantile_over_time_readouts, query, scan_group, sketch_state, - try_candidate, + try_candidate, with_series, }; use asap_types::test_support::memo_fixtures; use asap_types::test_support::memo_fixtures::arithmetic; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, SourceFacts}; use asap_types::workload::{ Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, TimestampMs, @@ -41,6 +47,7 @@ use crate::milp::accuracy::{AccuracyRules, AccuracyVariables, ErrorModel}; use crate::plan::plan_of; use crate::retention::{edge_retention_need_ms, least_retention_ms_per_execution}; use crate::selectable::{QueryId, SelectableEdge}; +use asap_accuracy::{readout_levels, ErrorMetricsOfMemo, NoMeasuredGuarantees}; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); @@ -64,7 +71,8 @@ fn empty_memo() -> Memo { /// Stage 2's output over `memo`, with one query-time candidate per /// (logical candidate, lifetime) of `candidates` and every edge between -/// a candidate of an input group and its consumer admissible. +/// a candidate of an input group and its consumer admissible, and each +/// query's failure-probability bound as Pass 2 records it. fn physical( memo: Memo, candidates: &[(CandidateForMemoGroupId, Lifetime)], @@ -102,6 +110,18 @@ fn physical( } } let groups = memo.groups().len(); + let readout_values_of_query = memo + .topological_order() + .map(|order| { + readout_levels( + &memo, + &ErrorMetricsOfMemo::of(&memo, &order), + &order, + &NoMeasuredGuarantees, + ) + .readout_values_of_query + }) + .unwrap_or_default(); PhysicalASAPMemoCandidates::new( LogicalASAPMemoCandidates { memo, @@ -110,6 +130,7 @@ fn physical( pruned: vec![], evaluation_grid_of_group: vec![None; groups], exact_only_groups_of_query: vec![], + readout_values_of_query, }, physical_candidates, edges, @@ -1225,15 +1246,31 @@ fn ddsketch_readouts( q: f64, levels: &[(f64, f64)], ) -> (MemoGroupId, Vec) { - let m = scan(memo); - let window = range(memo, 60, m); - let state = sketch_state( + readouts_of( memo, - window, + "m", SketchAlgorithm::DDSketch, - Reduction::PerEntity, - ); - let operator = memo_fixtures::readout(SketchStatistic::Quantile { q }, state); + SketchStatistic::Quantile { q }, + AccuracyMetric::Relative, + levels, + ) +} + +/// `statistic` over a state of `algorithm` over `metric[1m]` per series: +/// the readout group, one readout per requirement in `error_metric` of +/// `levels` (ε, δ). +fn readouts_of( + memo: &mut Memo, + metric: &str, + algorithm: SketchAlgorithm, + statistic: SketchStatistic, + error_metric: AccuracyMetric, + levels: &[(f64, f64)], +) -> (MemoGroupId, Vec) { + let rows = scan_group(memo, metric, &["job"], vec![]); + let window = range(memo, 60, rows); + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let operator = memo_fixtures::readout(statistic, state); let group = memo.find_or_add_group(operator.clone()).unwrap(); let readouts = levels .iter() @@ -1246,7 +1283,7 @@ fn ddsketch_readouts( ); readout.input_requirements = vec![LogicalRequirements { required_accuracy: Some(RequiredAccuracy { - metric: AccuracyMetric::Relative, + metric: error_metric, max_error, failure_probability, }), @@ -1257,6 +1294,25 @@ fn ddsketch_readouts( (group, readouts) } +/// A memo over sources each declaring only its series count. +fn memo_with_series(series: &[(&str, u64)]) -> Memo { + Memo::new(&DataWorkload { + sources: series + .iter() + .map(|&(metric, series_count)| { + ( + metric.to_string(), + SourceFacts { + series_count: Some(series_count), + ..SourceFacts::default() + }, + ) + }) + .collect(), + ..DataWorkload::default() + }) +} + /// Every candidate of `memo` at query time, `OneEvaluation`. fn every_candidate(memo: &Memo) -> Vec<(CandidateForMemoGroupId, Lifetime)> { memo.groups() @@ -1297,11 +1353,21 @@ fn relative(max_error: f64) -> AccuracyTarget { /// and misses the target, so it bounds nothing: as a bound it would leave /// out each 0.01 readout (cost 3 > 2). The same ratio every 10 s and every /// 20 s is one context's copy, whose (K) has no rows of its own and -/// selects alike. +/// selects alike. The ratio's only variable is its error $E$: the target +/// sets no failure probability, so no root checks one, with or without +/// `m`'s series declared. #[test] fn k_allocates_a_ratio_s_error_among_its_readouts() { - for entries in [vec![every(10_000)], vec![every(10_000), every(20_000)]] { - let mut memo = empty_memo(); + for (entries, series_declared) in [ + (vec![every(10_000)], false), + (vec![every(10_000), every(20_000)], false), + (vec![every(10_000)], true), + ] { + let mut memo = if series_declared { + memo_with_series(&[("m", 1)]) + } else { + empty_memo() + }; let levels = [(0.02, 1.0), (0.01, 1.0)]; let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); @@ -1344,7 +1410,7 @@ fn k_allocates_a_ratio_s_error_among_its_readouts() { ); assert!(!selected.contains(¬_chosen), "{selected:?}"); } - // (K)'s E and D exist only in the 10 s context (the 20 s one is + // (K)'s variables exist only in the 10 s context (the 20 s one is // its copy), and only for the ratio: a readout's error is fixed. let kept = vec![true; selectable.executions().len()]; let rules = AccuracyRules::of(&inputs).unwrap(); @@ -1359,7 +1425,7 @@ fn k_allocates_a_ratio_s_error_among_its_readouts() { ); let model = ErrorModel::new(&inputs, &rules, &variables.y, &shared); let accuracy = AccuracyVariables::new(&model, &mut problem); - let one_context = 2; + let one_context = 1; assert_eq!(accuracy.variable_count(), one_context, "{entries:?}"); } } @@ -1404,21 +1470,6 @@ fn k_accepts_error_only_in_the_target_s_metric() { assert!(plans(Some(any), &level, Some(1), operator)); } -/// A whole-result failure probability 0.01 over 10 values is 1e-3 per -/// value, which a readout failing with probability 1e-3 meets; over 100 -/// values (1e-4 per value) it does not. With the count unknown no -/// per-value δ follows, and (K) has no row for it (Pass 2 gives such a -/// query's readouts no level). -#[test] -fn k_divides_a_whole_result_failure_probability_over_the_values() { - let level = [(0.01, 1e-3)]; - let target = relative(0.01).failure_probability(0.01); - let operator = QueryRoot::Operator; - assert!(plans(Some(target.clone()), &level, Some(10), operator)); - assert!(!plans(Some(target.clone()), &level, Some(100), operator)); - assert!(plans(Some(target), &level, None, operator)); -} - /// `scalar(v)` returns the value it reads, so its root accepts the /// target's metric; `scalar(v) * 2`'s operand is read exactly, as no error /// contract covers the scalar expression; without a target either reads @@ -1468,8 +1519,9 @@ fn k_caps_a_ratio_s_denominator_at_its_validity_cap() { /// 0.01 over the whole result, returning 10 values every 10 s and 100 /// every 20 s: the readout failing with 1e-3 (cost 1) meets 1e-3 per /// value, not 1e-4, where the one failing with 1e-5 (cost 2) is needed. -/// The contexts differ in what (K) reads ($n_q$), so the 20 s one is not a -/// copy: each selects its own readout. +/// The series are undeclared, so each checks each covered value, 10 at +/// 1e-3 and 100 at 1e-4: the contexts differ in their bounds, so the 20 s +/// one is not a copy and each selects its own readout. #[test] fn contexts_whose_queries_split_their_failure_probability_differently_are_not_copies() { let mut memo = empty_memo(); @@ -1517,6 +1569,612 @@ fn contexts_whose_queries_split_their_failure_probability_differently_are_not_co ); } +/// `q` = p90 over 1,000 series, read per value at δ = 0.5, and `max(q)` +/// over the whole result at δ = 0.01, with one readout group holding a +/// copy failing with 5e-4 (cost 1, the level `q` alone needs) and one +/// failing with 1e-5 (cost 3). `max(q)` may fail when any of the 1,000 +/// estimates it reads does, so the 5e-4 copy gives it a failure +/// probability up to 1,000 × 5e-4 = 0.5, not 0.01: only the 1e-5 copy +/// (1,000 × 1e-5 = 0.01) meets its target. A rule adding the selected +/// readout's δ once accepts the 5e-4 copy for both queries. The same with +/// `q` per value at δ = 0.01, whose own level is 0.01 when δ splits over +/// the estimates one covered value depends on (1,000 × 0.01 = 10 for +/// `max(q)`). Either way `q` is cheapest reading the 1e-5 copy too, so +/// only that copy is selected. +#[test] +fn k_does_not_let_a_reduction_read_a_level_given_for_a_per_value_query() { + for (per_value_delta, per_value_level) in [(0.5, 5e-4), (0.01, 0.01)] { + let mut memo = Memo::new(&with_series("m", 1_000)); + let (readout_group, readouts) = + ddsketch_readouts(&mut memo, 0.9, &[(0.01, per_value_level), (0.01, 1e-5)]); + let max = group( + &mut memo, + memo_fixtures::aggregate( + Reduction::by(vec![]), + AggIntent::Max { col: None }, + readout_group, + ), + ); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some( + relative(0.01) + .failure_probability(per_value_delta) + .accuracy_scope(AccuracyScope::PerValue), + ), + number_of_values_per_query_evaluation: Some(1_000), + }); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(max), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + // The scan costs 10, so neither readout is dearer than a feasible + // plan. + let mut w = vec![0.0; physical.physical_candidates.len()]; + w[0] = 10.0; + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + let (solved, _) = solve(&physical, &every_10s(2), &w, &[], &[]); + let selected = candidates(&solved.unwrap()); + assert!( + selected.contains(&physical_of(&physical, readouts[1])), + "{per_value_delta}: {selected:?}" + ); + } +} + +/// Whether a plan meets (K) for `memo`'s queries, every one every 10 s, +/// with every candidate at cost 1 but the scan of `m` (candidate 0) at +/// 10: the selected candidates, or `None` when (K) is infeasible. +fn selected_or_infeasible(memo: &Memo) -> Option> { + let physical = physical(memo.clone(), &every_candidate(memo)); + let mut w = vec![1.0; physical.physical_candidates.len()]; + w[0] = 10.0; + match solve(&physical, &every_10s(memo.queries().len()), &w, &[], &[]).0 { + Ok(solved) => Some(candidates(&solved)), + Err(SelectionMilpError::Infeasible) => None, + Err(other) => panic!("{other:?}"), + } +} + +/// A query rooted at `root` with `target` and $n_q$ `values`. +fn add_query(memo: &mut Memo, root: MemoGroupId, target: AccuracyTarget, values: Option) { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(root), + accuracy_target: Some(target), + number_of_values_per_query_evaluation: values, + }); +} + +/// `q or r`, p90 of `m` and of `r`, 100 series each with the same labels, +/// over a whole result of $n_q$ = 100 at δ = 0.01: each output value is +/// copied from one readout value, so the statement fails only when one of +/// at most 100 estimates does. With both readouts at 1e-4 (Pass 2's split, +/// 0.01 / min(200, 100 · 1)), each covered value fails with at most +/// max(1e-4, 1e-4), and 100 of them with 0.01: (K) accepts it. At 2e-4 +/// each, 0.02: it does not. Adding both sides' δ per value (2e-4 against +/// 1e-4) would leave `or` no sketch plan. +#[test] +fn k_accepts_an_or_whose_readouts_meet_delta_over_its_covered_values() { + for (level, plans) in [(1e-4, true), (2e-4, false)] { + let mut memo = memo_with_series(&[("m", 100), ("r", 100)]); + let p90 = SketchStatistic::Quantile { q: 0.9 }; + let relative_metric = AccuracyMetric::Relative; + let (of_m, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::DDSketch, + p90.clone(), + relative_metric, + &[(0.05, level)], + ); + let (of_r, _) = readouts_of( + &mut memo, + "r", + SketchAlgorithm::DDSketch, + p90, + relative_metric, + &[(0.05, level)], + ); + let or = group( + &mut memo, + memo_fixtures::set_operator(PromQLVectorSetOpKind::Or, of_m, of_r), + ); + add_query( + &mut memo, + or, + relative(0.05).failure_probability(0.01), + Some(100), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `q * on() group_left e`: `q` exact over 1000 series of `m`, `e` the p90 +/// of one series of `e`, over the whole result of 1000 values at δ = +/// 0.01. Every output value reads `e`'s one estimate, so the result fails +/// only when that estimate does: a readout failing with 0.01 meets it, +/// one failing with 0.02 does not. Splitting δ over the 1000 output +/// values (1e-5 each) would leave it no sketch plan. +#[test] +fn k_charges_a_readout_every_series_matches_once() { + for (level, plans) in [(0.01, true), (0.02, false)] { + let mut memo = memo_with_series(&[("m", 1_000), ("e", 1)]); + let rows = scan(&mut memo); + let window = range(&mut memo, 60, rows); + let q = group( + &mut memo, + memo_fixtures::aggregate(Reduction::PerEntity, AggIntent::Max { col: None }, window), + ); + let (e, _) = readouts_of( + &mut memo, + "e", + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.9 }, + AccuracyMetric::Relative, + &[(0.01, level)], + ); + let matched = group( + &mut memo, + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec![], + grouping: Some(VectorGrouping { + side: GroupSide::Left, + labels: vec![], + }), + }), + }, + return_bool: false, + lhs: q, + rhs: e, + }), + ); + add_query( + &mut memo, + matched, + relative(0.05).failure_probability(0.01), + Some(1_000), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `q and r` reads only which series `r` has, and `q / count(r)` none of +/// `r`'s values (β = 0 on `r`), so `r`'s readout, failing with 0.5 at a +/// level another query gave it, cannot make the output miss: over one +/// series at δ = 0.01, `q`'s readout at 0.01 meets the target alone. +#[test] +fn k_adds_nothing_for_an_input_whose_values_are_not_read() { + for division in [false, true] { + let mut memo = memo_with_series(&[("m", 1)]); + let (q, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.01, 0.01)]); + let (r, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 0.5)]); + let root = if division { + let count = group( + &mut memo, + memo_fixtures::aggregate(Reduction::by(vec![]), AggIntent::Count, r), + ); + group(&mut memo, arithmetic(ArithmeticOpKind::Div, q, count)) + } else { + group( + &mut memo, + memo_fixtures::set_operator(PromQLVectorSetOpKind::And, q, r), + ) + }; + add_query( + &mut memo, + root, + relative(0.05).failure_probability(0.01), + Some(1), + ); + assert!(selected_or_infeasible(&memo).is_some(), "{division}"); + } +} + +/// `q + r` with `q` exact and `r` a p90 readout at 0.01 (absolute error, +/// which a sum passes on), one series each: +/// an exact input has no estimate to fail, so the sum fails only with +/// `r`, 0.01, within δ = 0.01. The sum is not constant (one of its inputs +/// is a choice), so it has an error bound $E$ and the one failure +/// probability its root checks: `R` = $V$ · fan-in = 1 is a tie, so every +/// estimate read, $W$. Two variables. +#[test] +fn k_counts_an_exact_input_as_never_failing() { + let mut memo = memo_with_series(&[("m", 1)]); + let rows = scan(&mut memo); + let window = range(&mut memo, 60, rows); + let q = group( + &mut memo, + memo_fixtures::aggregate(Reduction::PerEntity, AggIntent::Max { col: None }, window), + ); + let (r, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.9 }, + AccuracyMetric::ABSOLUTE, + &[(0.01, 0.01)], + ); + let sum = group(&mut memo, arithmetic(ArithmeticOpKind::Add, q, r)); + add_query( + &mut memo, + sum, + AccuracyTarget::max_error(0.05) + .accuracy_metric(AccuracyMetric::ABSOLUTE) + .failure_probability(0.01), + Some(1), + ); + assert!(selected_or_infeasible(&memo).is_some()); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let entries = every_10s(1); + let (selectable, costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs).unwrap(); + let shared = copies_in_equivalent_contexts(&inputs, &kept, Some(&rules)); + let mut problem = ProblemVariables::new(); + let variables = selection_variables( + &inputs, + &kept, + &shared, + &Components::of(&inputs), + &mut problem, + ); + let model = ErrorModel::new(&inputs, &rules, &variables.y, &shared); + assert_eq!( + AccuracyVariables::new(&model, &mut problem).variable_count(), + 2 + ); +} + +/// A readout over 100 series read per value at δ = 0.01: each value fails +/// only with its own estimate, so the readout at 0.01 meets it, whatever +/// the 100 values; one at 0.02 does not. +#[test] +fn k_does_not_divide_a_per_value_failure_probability_by_the_values() { + for (level, plans) in [(0.01, true), (0.02, false)] { + let mut memo = memo_with_series(&[("m", 100)]); + let (readout, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, level)]); + add_query( + &mut memo, + readout, + relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(100), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// A readout failing with 1e-3 read over the whole result at δ = 0.01. +/// With its series undeclared, (K) checks each covered value at δ / $n_q$: +/// 1e-3 over 10 values (10 · 1e-3), not over 100. With $n_q$ unknown too +/// no bound is known: (K) has no row, and the query reads the readout +/// group only exactly, by (X), so with only readouts in it no plan +/// exists. With its 10 series declared and $n_q$ unknown, every estimate +/// read bounds it: 10 · 1e-3 = 0.01. +#[test] +fn k_bounds_a_whole_result_by_its_covered_values_or_every_estimate() { + let target = relative(0.01).failure_probability(0.01); + let plans_over = |memo: Memo, values: Option, exact_only: bool| { + let mut memo = memo; + let (readout, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1e-3)]); + add_query(&mut memo, readout, target.clone(), values); + let mut physical = physical(memo.clone(), &every_candidate(&memo)); + if exact_only { + // (X) lists the readout group and every group below it. + let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = + memo.groups()[readout.0 as usize].expression + else { + unreachable!("a readout group") + }; + physical.logical_candidates.exact_only_groups_of_query = + vec![[readout, summary_input].into()]; + } + let w = vec![1.0; physical.physical_candidates.len()]; + match solve(&physical, &every_10s(1), &w, &[], &[]).0 { + Ok(_) => true, + Err(SelectionMilpError::Infeasible) => false, + Err(other) => panic!("{other:?}"), + } + }; + assert!(plans_over(empty_memo(), Some(10), false)); + assert!(!plans_over(empty_memo(), Some(100), false)); + assert!(!plans_over(empty_memo(), None, true)); + assert!(plans_over(memo_with_series(&[("m", 10)]), None, false)); +} + +/// A readout over 1000 series read over the groups holding at least 5% of +/// the records, at δ = 0.01: at most 20 groups qualify, so (K) checks +/// each of them at 0.01 / 20 = 5e-4. A readout at 5e-4 meets it; one at +/// 1e-3 does not. +#[test] +fn k_bounds_a_record_share_scope_by_its_covered_groups() { + for (level, plans) in [(5e-4, true), (1e-3, false)] { + let mut memo = memo_with_series(&[("m", 1_000)]); + let (readout, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, level)]); + add_query( + &mut memo, + readout, + relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::GroupsWithRecordShareAtLeast(0.05)), + Some(1_000), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `readout or {job="a"}(readout)`, the same p90 readout of 100 series on +/// both sides, over the whole result of 100 values at δ = 0.01: each +/// output value copies one estimate, so the readout at 1e-4 meets each +/// covered value (max(1e-4, 1e-4), 100 values: 0.01), although its two +/// paths read 200 estimates' worth. +#[test] +fn k_charges_an_estimate_both_sides_of_an_or_read_once_per_value() { + let mut memo = memo_with_series(&[("m", 100)]); + let (readout, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.05, 1e-4)]); + let or = or_of_readout_and_its_filter(&mut memo, readout); + add_query( + &mut memo, + or, + relative(0.05).failure_probability(0.01), + Some(100), + ); + assert!(selected_or_infeasible(&memo).is_some()); +} + +/// `readout or {job="a"}(readout)`. +fn or_of_readout_and_its_filter(memo: &mut Memo, readout: MemoGroupId) -> MemoGroupId { + let job_a = memo_fixtures::compare( + 2, + CompareOpKind::Eq, + ScalarValue::Utf8("a".into()), + ExprSemantics::Promql, + ); + let filtered = group(memo, memo_fixtures::filter(job_a, readout)); + group( + memo, + memo_fixtures::set_operator(PromQLVectorSetOpKind::Or, readout, filtered), + ) +} + +/// `max(readout or {job="a"}(readout))` over 100 series at δ = 0.01: the +/// max reads every estimate of both sides, 200 by path, so (K) checks +/// every estimate read and charges each estimate once per path: the +/// readout at 1e-4 gives 200 · 1e-4 = 0.02, and the dearer one at 5e-5 +/// (0.01) is selected. Only 100 distinct estimates can fail, so 1e-4 +/// would have been enough (`accuracy-models.md`, "What the rows lose", +/// item 1): a stricter δ than needed, never a missed target. +#[test] +fn k_charges_an_estimate_two_paths_reach_once_per_path() { + let mut memo = memo_with_series(&[("m", 100)]); + let (readout, readouts) = ddsketch_readouts(&mut memo, 0.9, &[(0.05, 1e-4), (0.05, 5e-5)]); + let or = or_of_readout_and_its_filter(&mut memo, readout); + let max = group( + &mut memo, + memo_fixtures::aggregate(Reduction::by(vec![]), AggIntent::Max { col: None }, or), + ); + add_query( + &mut memo, + max, + relative(0.05).failure_probability(0.01), + Some(1), + ); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let mut w = vec![0.0; physical.physical_candidates.len()]; + w[0] = 10.0; + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 2.0; + let selected = candidates(&solve(&physical, &every_10s(1), &w, &[], &[]).0.unwrap()); + assert!( + selected.contains(&physical_of(&physical, readouts[1])), + "{selected:?}" + ); + assert!( + !selected.contains(&physical_of(&physical, readouts[0])), + "{selected:?}" + ); +} + +/// A top-k readout over 10 series, read per value at δ = 0.01: its output +/// keys are chosen by all 10 estimates it ranks, so one output value +/// fails when any of them does. The readout at 1e-3 meets it (10 · 1e-3), +/// at 2e-3 it does not. +#[test] +fn k_charges_a_top_k_readout_for_every_estimate_it_ranks() { + for (level, plans) in [(1e-3, true), (2e-3, false)] { + let mut memo = memo_with_series(&[("m", 10)]); + let (top_k, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 3 }, + AccuracyMetric::Relative, + &[(0.01, level)], + ); + add_query( + &mut memo, + top_k, + AccuracyTarget::max_error(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(30), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// The top-k readout of 10 series of `m` (CMS with heap, k = 3) failing +/// with `top_k_level` per estimate, and the p90 readout of 10 series of +/// `r` failing with `p90_level`, combined by the set operator `kind` with +/// the p90 readout on the left, read per value at δ = 0.01; `or` with the +/// top-k on the left. `None` when no plan meets the target. +fn set_operator_over_top_k( + kind: PromQLVectorSetOpKind, + top_k_level: f64, + p90_level: f64, +) -> Option> { + let mut memo = memo_with_series(&[("m", 10), ("r", 10)]); + let (top_k, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 3 }, + AccuracyMetric::Relative, + &[(0.01, top_k_level)], + ); + let (p90, _) = readouts_of( + &mut memo, + "r", + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.9 }, + AccuracyMetric::Relative, + &[(0.01, p90_level)], + ); + let (left, right) = if matches!(kind, PromQLVectorSetOpKind::Or) { + (top_k, p90) + } else { + (p90, top_k) + }; + let set = group(&mut memo, memo_fixtures::set_operator(kind, left, right)); + add_query( + &mut memo, + set, + relative(0.05) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(10), + ); + selected_or_infeasible(&memo) +} + +/// `p90 and topk(3, …)` and `p90 unless topk(3, …)` keep a p90 value +/// only where the top-k's output has (or lacks) its labels, and which +/// keys the top-k returns is chosen by all 10 estimates it ranks: each +/// output value fails when its p90 estimate or any of those does. The +/// top-k at 1e-3 (10 · 1e-3 = 0.01) with the p90 at 0.005 sums to 0.015, +/// above δ = 0.01; at 1e-4 (0.001) the sum is 0.006, within it. +/// `topk(3, …) or p90` takes a p90 value only where the top-k returns no +/// such key, so each value depends on both sides' estimates: the same +/// sums, not the larger side. +#[test] +fn k_charges_a_set_operator_for_the_top_k_choosing_its_values() { + for kind in [ + PromQLVectorSetOpKind::And, + PromQLVectorSetOpKind::Unless, + PromQLVectorSetOpKind::Or, + ] { + for (top_k_level, plans) in [(1e-3, false), (1e-4, true)] { + assert_eq!( + set_operator_over_top_k(kind.clone(), top_k_level, 0.005).is_some(), + plans, + "{kind:?} {top_k_level}" + ); + } + } +} + +/// The p90 and the p50 readouts of 100 series concatenated, per value at +/// δ = 0.01: each row comes from one readout, so it fails only with that +/// readout's estimate: readouts at 0.01 meet it, at 0.02 they do not. +#[test] +fn k_charges_a_concatenation_one_input_per_value() { + for (level, plans) in [(0.01, true), (0.02, false)] { + let mut memo = memo_with_series(&[("m", 100)]); + let (p90, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.01, level)]); + let (p50, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, level)]); + let both = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Concat { + children: vec![p90, p50], + }), + ); + add_query( + &mut memo, + both, + relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(200), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `scalar(p90) + scalar(p50)` over one series at δ = 0.01, each readout +/// required with no error (the operands of a scalar expression no +/// contract covers are read exactly) but failing with `level`: the sum +/// fails when either does, so its one row adds both roots' failure +/// probabilities. 0.005 each meets 0.01; 0.006 each (0.012) does not, +/// although each root alone is within 0.01. +#[test] +fn k_sums_the_failure_probabilities_of_a_query_s_roots() { + for (level, plans) in [(0.005, true), (0.006, false)] { + let mut memo = memo_with_series(&[("m", 1)]); + let (p90, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.0, level)]); + let (p50, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.0, level)]); + memo.add_query(MemoQuery { + root: QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Add, + left: Box::new(ScalarExpr::PromqlScalarFromVector(p90)), + right: Box::new(ScalarExpr::PromqlScalarFromVector(p50)), + semantics: ExprSemantics::Promql, + }), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// The same readout over 10 series every 10 s and every 20 s, declaring +/// 10 and 50 values, both over the whole result at δ = 0.01: each reads +/// its 10 estimates (no more than its values), so both check every +/// estimate read against 0.01, and the 20 s context is a copy. +#[test] +fn contexts_whose_queries_bound_every_estimate_read_at_equal_delta_are_copies() { + let mut memo = memo_with_series(&[("m", 10)]); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1e-3)]); + for values in [10, 50] { + add_query( + &mut memo, + readout_group, + relative(0.01).failure_probability(0.01), + Some(values), + ); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let entries = [every(10_000), every(20_000)]; + let (selectable, costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs); + let shared = copies_in_equivalent_contexts(&inputs, &kept, rules.as_ref()); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(0)]); +} + /// The same readout group as `q` every 10 s and as `scalar(q) * 2` every /// 20 s, both at relative 0.01, with only a readout at relative 0.01: the /// operator root accepts it, the scalar expression's operand only exact @@ -1656,20 +2314,26 @@ fn a_selection_missing_its_target_at_the_default_tolerance_is_solved_again_at_th /// first solve selects two (1.08e-6, within HiGHS's default tolerance of /// the 1e-6 limit), the exact check finds the miss, and at tolerance 1e-10 /// no plan meets the limit, so HiGHS reports the MILP infeasible. The -/// error names the query the first solve missed and its per-value δ, -/// not a plan-less "infeasible" that never mentions accuracy. +/// error names the query the first solve missed and its bound: each of +/// the 1e4 covered values at 1e-6 (the readouts' values per evaluation +/// are unknown, so every estimate read bounds nothing), not a plan-less +/// "infeasible" that never mentions accuracy. #[test] fn a_target_no_plan_meets_once_solved_at_the_least_tolerance_names_the_missed_query() { let solved = solve_ratio_over_values(10_000, [5.4e-7, 5.4e-7]); let Err(SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_per_output_value: Some(delta), + failure_probability_bound: + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 10_000, + limit, + }), }) = solved else { panic!("{solved:?}"); }; assert_eq!(query, QueryId(0)); - assert!((delta - 1e-6).abs() <= 1e-12, "{delta}"); + assert!((limit - 1e-6).abs() <= 1e-12, "{limit}"); } /// The same ratio over readouts failing with 4e-7 (cost 1) or 3e-7 @@ -1697,12 +2361,14 @@ fn a_per_value_failure_probability_below_the_solver_precision_is_an_error() { let solved = solve_ratio_over_values(values, failure_probabilities); let Err(SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_per_output_value: Some(delta), + failure_probability_bound: Some(FailureProbabilityBound::EachCoveredValue { limit, .. }), }) = solved else { panic!("{values}: {solved:?}"); }; assert_eq!(query, QueryId(0)); - assert!((delta - per_value).abs() <= 1e-6 * per_value, "{delta}"); + assert!((limit - per_value).abs() <= 1e-6 * per_value, "{limit}"); } } + +mod union_bound_oracle; diff --git a/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs b/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs new file mode 100644 index 000000000..d3c8a460e --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs @@ -0,0 +1,596 @@ +//! A brute-force check of constraint (K)'s failure-probability rows +//! (`accuracy-models.md`, "Which estimates a result depends on"): for +//! every plan the MILP accepts, the union-bound sum over the estimates the +//! covered output values depend on, maximized over every dataset the +//! declared facts allow, is at most the query's δ. +//! +//! It computes the union-bound worst case, `max_D Σ_{e ∈ S_q(D)} δ_e`, not +//! the true failure probability (which also depends on how the estimates' +//! failures correlate); that sum is what the rows must bound. +//! +//! Each shape is built over two metrics `m` and `b`, each declaring 6 +//! series (labels `job` ∈ {0, 1} and `instance` ∈ {0, 1, 2}); a dataset is +//! a subset of each metric's 6 series (64 × 64 datasets). Each query's +//! selected path is evaluated over every dataset, every output value +//! carrying the estimates (readout copy, series) it depends on: +//! - a readout's value of a series depends on that estimate; +//! - an exact per-series value on none; +//! - a filter keeps `job = 0`; +//! - `or` copies the left value, else the right one; `and` and `unless` +//! copy the left value where the right does (not) have the series; +//! - `+` one to one unions both sides' sets; `+ on() group_left` adds the +//! right side's only value (no output unless it has exactly one); +//! - `max` and `avg by (job)` union their group's sets. +//! +//! Each readout group gets the level Pass 2's split gives each query +//! reaching it (`AccuracyTarget::failure_probability_per_readout_value` +//! over the counts `asap_accuracy::readout_levels` records) and random +//! further copies, some looser and cheaper, so the solver is tempted by +//! copies given for another query. Queries are planned alone and in pairs +//! (a per-value query on a readout with a query over a shape reading it), +//! under the whole-result, per-value and record-share scopes. + +use std::collections::{BTreeMap, BTreeSet}; + +use asap_accuracy::{readout_levels, ErrorMetricsOfMemo, NoMeasuredGuarantees}; +use asap_types::workload::bounds::source_name; + +use super::*; + +/// A series of `m` or `b`: (job, instance). +type Series = (u8, u8); + +/// An output value's key: a series, a job, or nothing (a global +/// reduction). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Key { + Series(Series), + Job(u8), + Nothing, +} + +/// An estimate: the readout execution's physical candidate and the +/// series. +type Estimate = (usize, Series); + +/// One evaluation's output: per key, the estimates the value depends on. +type Values = BTreeMap>; + +const SERIES: [Series; 6] = [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)]; + +/// A small deterministic generator (SplitMix64). +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// A PromQL set operator. +#[derive(Debug, Clone, Copy)] +enum SetKind { + Or, + And, + Unless, +} + +impl SetKind { + fn operator(self) -> PromQLVectorSetOpKind { + match self { + Self::Or => PromQLVectorSetOpKind::Or, + Self::And => PromQLVectorSetOpKind::And, + Self::Unless => PromQLVectorSetOpKind::Unless, + } + } +} + +/// The query shapes, as builders over a memo. +#[derive(Debug, Clone, Copy)] +enum Shape { + Readout(&'static str), + Exact(&'static str), + Filter(&'static Shape), + Max(&'static Shape), + AvgByJob(&'static Shape), + Set(SetKind, &'static Shape, &'static Shape), + Add(&'static Shape, &'static Shape), + /// `left + on() group_left max(right)`. + AddMaxOfRight(&'static Shape, &'static Shape), +} + +const RM: Shape = Shape::Readout("m"); +const RB: Shape = Shape::Readout("b"); +const EM: Shape = Shape::Exact("m"); +const FILTER_RM: Shape = Shape::Filter(&RM); +const OR: Shape = Shape::Set(SetKind::Or, &RM, &RB); +const OR_SELF: Shape = Shape::Set(SetKind::Or, &RM, &FILTER_RM); +const AND: Shape = Shape::Set(SetKind::And, &RM, &RB); +const UNLESS: Shape = Shape::Set(SetKind::Unless, &RM, &RB); +const ADD: Shape = Shape::Add(&RM, &RB); +const ADD_EXACT: Shape = Shape::Add(&EM, &RB); +const GROUP_LEFT: Shape = Shape::AddMaxOfRight(&EM, &RB); +const GROUP_LEFT_READOUTS: Shape = Shape::AddMaxOfRight(&RM, &RB); + +/// Every shape of depth at most 3 the oracle covers. +const SHAPES: &[Shape] = &[ + RM, + FILTER_RM, + Shape::Max(&RM), + Shape::AvgByJob(&RM), + OR, + OR_SELF, + AND, + UNLESS, + ADD, + ADD_EXACT, + GROUP_LEFT, + GROUP_LEFT_READOUTS, + Shape::Max(&OR), + Shape::Max(&OR_SELF), + Shape::AvgByJob(&OR), + Shape::Max(&ADD), + Shape::AvgByJob(&ADD_EXACT), + Shape::Max(&AND), + Shape::Filter(&OR), + Shape::Set(SetKind::Or, &ADD, &RB), + Shape::Add(&OR, &RB), +]; + +/// The column of label `label` in `group`'s schema. +fn column_of(memo: &Memo, group: MemoGroupId, label: &str) -> usize { + memo.groups()[group.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == label) + .unwrap() +} + +/// Builds `shape` in `memo`; the readout groups it adds are listed in +/// `readouts` (each with one candidate without a requirement, as Pass 1 +/// leaves it). +fn build(memo: &mut Memo, shape: Shape, readouts: &mut BTreeSet) -> MemoGroupId { + let built = match shape { + Shape::Readout(metric) => { + let rows = scan_group(memo, metric, &["job", "instance"], vec![]); + let window = range(memo, 60, rows); + let state = sketch_state( + memo, + window, + SketchAlgorithm::DDSketch, + Reduction::PerEntity, + ); + let operator = memo_fixtures::readout(SketchStatistic::Quantile { q: 0.9 }, state); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + if readouts.insert(group) { + try_candidate(memo, group, operator, None).unwrap(); + } + return group; + } + Shape::Exact(metric) => { + let rows = scan_group(memo, metric, &["job", "instance"], vec![]); + let window = range(memo, 60, rows); + memo_fixtures::aggregate(Reduction::PerEntity, AggIntent::Max { col: None }, window) + } + Shape::Filter(child) => { + let child = build(memo, *child, readouts); + let job_0 = memo_fixtures::compare( + column_of(memo, child, "job"), + CompareOpKind::Eq, + ScalarValue::Utf8("0".into()), + ExprSemantics::Promql, + ); + memo_fixtures::filter(job_0, child) + } + Shape::Max(child) => { + let child = build(memo, *child, readouts); + memo_fixtures::aggregate(Reduction::by(vec![]), AggIntent::Max { col: None }, child) + } + Shape::AvgByJob(child) => { + let child = build(memo, *child, readouts); + let job = column_of(memo, child, "job"); + memo_fixtures::aggregate( + Reduction::by(vec![job]), + AggIntent::Avg { col: None }, + child, + ) + } + Shape::Set(kind, left, right) => { + let (left, right) = (build(memo, *left, readouts), build(memo, *right, readouts)); + memo_fixtures::set_operator(kind.operator(), left, right) + } + Shape::Add(left, right) => { + let (left, right) = (build(memo, *left, readouts), build(memo, *right, readouts)); + arithmetic(ArithmeticOpKind::Add, left, right) + } + Shape::AddMaxOfRight(left, right) => { + let left = build(memo, *left, readouts); + let right = build(memo, Shape::Max(right), readouts); + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec![], + grouping: Some(VectorGrouping { + side: GroupSide::Left, + labels: vec![], + }), + }), + }, + return_bool: false, + lhs: left, + rhs: right, + }) + } + }; + group(memo, built) +} + +/// What one evaluation of execution `v` returns over `dataset` (the +/// series of each metric present), with `producer_of` the selected +/// producer per (consumer, input). +fn evaluate( + physical: &PhysicalASAPMemoCandidates, + producer_of: &BTreeMap<(usize, usize), usize>, + v: usize, + dataset: &BTreeMap<&str, Vec>, +) -> Values { + let memo = &physical.logical_candidates.memo; + let id = physical.physical_candidates[v].logical_candidate_id; + let candidate = memo.candidate(id); + let input = |i: usize| evaluate(physical, producer_of, producer_of[&(v, i)], dataset); + let series_of_group = || { + let source = memo.groups()[id.group.0 as usize] + .group_properties + .scanned_sources + .iter() + .next() + .unwrap(); + dataset[source_name(source)].clone() + }; + match &candidate.expression.operator { + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) => series_of_group() + .into_iter() + .map(|series| (Key::Series(series), [(v, series)].into())) + .collect(), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + .. + }) => series_of_group() + .into_iter() + .map(|series| (Key::Series(series), BTreeSet::new())) + .collect(), + Operator::NonASAP(NonASAPOp::Filter { .. }) => input(0) + .into_iter() + .filter(|(key, _)| matches!(key, Key::Series((0, _)))) + .collect(), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(by), + .. + }) => { + let mut out = Values::new(); + for (key, estimates) in input(0) { + let group = match (by.keys().is_empty(), key) { + (true, _) => Key::Nothing, + (false, Key::Series((job, _)) | Key::Job(job)) => Key::Job(job), + (false, Key::Nothing) => Key::Nothing, + }; + out.entry(group).or_default().extend(estimates); + } + out + } + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + kind, vector_match, .. + }, + .. + }) => { + let (left, right) = (input(0), input(1)); + match (kind, vector_match) { + (BinaryOpKind::Set(PromQLVectorSetOpKind::Or), _) => { + let mut out = right; + out.extend(left); + out + } + (BinaryOpKind::Set(PromQLVectorSetOpKind::And), _) => left + .into_iter() + .filter(|(key, _)| right.contains_key(key)) + .collect(), + (BinaryOpKind::Set(PromQLVectorSetOpKind::Unless), _) => left + .into_iter() + .filter(|(key, _)| !right.contains_key(key)) + .collect(), + (_, Some(_)) => { + // `on() group_left`: the right side's only value. + let mut only = right.into_values(); + match (only.next(), only.next()) { + (Some(matched), None) => left + .into_iter() + .map(|(key, mut estimates)| { + estimates.extend(matched.iter().copied()); + (key, estimates) + }) + .collect(), + _ => Values::new(), + } + } + (_, None) => left + .into_iter() + .filter_map(|(key, mut estimates)| { + estimates.extend(right.get(&key)?.iter().copied()); + Some((key, estimates)) + }) + .collect(), + } + } + other => panic!("the oracle does not evaluate {other:?}"), + } +} + +/// The union-bound worst case of query `query`'s selected path at `v` +/// over every dataset: the largest summed δ of the estimates the values +/// its scope covers depend on. +fn worst_union_bound( + physical: &PhysicalASAPMemoCandidates, + producer_of: &BTreeMap<(usize, usize), usize>, + v: usize, + scope: AccuracyScope, +) -> f64 { + let memo = &physical.logical_candidates.memo; + let delta_of = |copy: usize| { + let id = physical.physical_candidates[copy].logical_candidate_id; + memo.candidate(id) + .required_accuracy() + .map_or(0.0, |required| required.failure_probability) + }; + let mut worst: f64 = 0.0; + for m_subset in 0u32..64 { + for b_subset in 0u32..64 { + let present = |subset: u32| -> Vec { + (0..6) + .filter(|bit| subset & (1 << bit) != 0) + .map(|bit| SERIES[bit]) + .collect() + }; + let dataset: BTreeMap<&str, Vec> = + [("m", present(m_subset)), ("b", present(b_subset))].into(); + let values: Vec> = evaluate(physical, producer_of, v, &dataset) + .into_values() + .collect(); + let sum = |covered: &[&BTreeSet]| -> f64 { + let union: BTreeSet = + covered.iter().flat_map(|set| set.iter().copied()).collect(); + union.iter().map(|&(copy, _)| delta_of(copy)).sum() + }; + let covered = match scope { + AccuracyScope::PerValue => values.iter().map(|set| sum(&[set])).fold(0.0, f64::max), + AccuracyScope::WholeResult => sum(&values.iter().collect::>()), + AccuracyScope::GroupsWithRecordShareAtLeast(share) => { + // Which groups qualify depends on exact record counts: + // any `⌊1 / share⌋` of them may. + let at_most = (1.0 / share).floor() as usize; + (0u32..1 << values.len()) + .filter(|chosen| chosen.count_ones() as usize <= at_most) + .map(|chosen| { + let subset: Vec<&BTreeSet> = values + .iter() + .enumerate() + .filter(|(index, _)| chosen & (1 << index) != 0) + .map(|(_, set)| set) + .collect(); + sum(&subset) + }) + .fold(0.0, f64::max) + } + }; + worst = worst.max(covered); + } + } + worst +} + +/// The most values one evaluation of `shape` returns over the datasets, +/// $n_q$ as Stage 0 would bound it (at least 1). +fn values_per_evaluation(shape: Shape) -> u64 { + match shape { + Shape::Readout(_) | Shape::Exact(_) | Shape::Filter(_) => 6, + Shape::Max(_) => 1, + Shape::AvgByJob(_) => 2, + Shape::Set(..) | Shape::Add(..) | Shape::AddMaxOfRight(..) => 6, + } +} + +/// One planning case: the queries (shape, scope), with `delta` each (no +/// failure probability when `None`, which only checks that a plan +/// exists). +fn check( + queries: &[(Shape, AccuracyScope)], + delta: Option, + rng: &mut Rng, + report: &mut OracleReport, +) { + let mut memo = memo_with_series(&[("m", 6), ("b", 6)]); + let mut readouts = BTreeSet::new(); + let mut roots = Vec::new(); + for &(shape, scope) in queries { + let root = build(&mut memo, shape, &mut readouts); + roots.push(root); + let mut target = AccuracyTarget::max_error(1.0) + .accuracy_metric(AccuracyMetric::ABSOLUTE) + .accuracy_scope(scope); + if let Some(delta) = delta { + target = target.failure_probability(delta); + } + add_query(&mut memo, root, target, Some(values_per_evaluation(shape))); + } + // Pass 2's level per query reaching each readout group, plus random + // copies at δ from 1e-5 to 0.5. + let order = memo.topological_order().unwrap(); + let readout_values_of_query = readout_levels( + &memo, + &ErrorMetricsOfMemo::of(&memo, &order), + &order, + &NoMeasuredGuarantees, + ) + .readout_values_of_query; + for &readout in &readouts { + let pass1 = CandidateForMemoGroupId { + group: readout, + index_in_group: 0, + }; + let mut levels: Vec = memo.groups()[readout.0 as usize] + .reached_by_queries() + .iter() + .filter_map(|&query| { + let memo_query = &memo.queries()[query]; + memo_query + .accuracy_target + .as_ref()? + .failure_probability_per_readout_value( + memo_query.number_of_values_per_query_evaluation, + readout_values_of_query[query]?, + ) + }) + .collect(); + for _ in 0..2 { + levels.push([1e-5, 1e-4, 1e-3, 5e-3, 0.01, 0.05, 0.5][rng.below(7) as usize]); + } + let unset = memo.candidate(pass1).clone(); + let requirement = |failure_probability| { + vec![LogicalRequirements { + required_accuracy: Some(RequiredAccuracy { + metric: AccuracyMetric::ABSOLUTE, + max_error: 0.01, + failure_probability, + }), + }] + }; + memo.set_candidate_input_requirements(pass1, requirement(levels[0])); + for &level in &levels[1..] { + let mut copy = unset.clone(); + copy.input_requirements = requirement(level); + let _ = memo.add_candidate(readout, copy); + } + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + // Random costs; a looser copy tends to be cheaper. + let w: Vec = physical + .physical_candidates + .iter() + .map(|candidate| { + let looseness = memo + .candidate(candidate.logical_candidate_id) + .required_accuracy() + .map_or(0.0, |required| required.failure_probability.log10() + 6.0); + 1.0 + rng.below(5) as f64 + 10.0 / (1.0 + looseness) + }) + .collect(); + let entries = every_10s(queries.len()); + let solved = match solve(&physical, &entries, &w, &[], &[]).0 { + Ok(solved) => solved, + Err(SelectionMilpError::Infeasible) => { + report.infeasible.push(format!("{queries:?}")); + return; + } + Err(other) => panic!("{queries:?}: {other:?}"), + }; + let (selectable, _) = selection_inputs(&physical, &entries, &w, &[], &[]); + let position_of = |execution: &Execution| { + selectable + .executions() + .iter() + .position(|known| known == execution) + .unwrap() + }; + let candidate_of = |position: usize| selectable.executions()[position].candidate.0; + let mut producer_of = BTreeMap::new(); + for edge in &solved.edges { + let input = physical.edge(edge.edge).input_index_of_consumer; + producer_of.insert( + (edge.consumer.candidate.0, input), + candidate_of(position_of(&edge.producer)), + ); + } + for (query, output) in solved.outputs.iter().enumerate() { + let (_, scope) = queries[query]; + let worst = worst_union_bound(&physical, &producer_of, output.candidate.0, scope); + report.accepted += 1; + if worst > 0.01 * (1.0 + 1e-9) { + report + .violations + .push(format!("{queries:?} query {query}: worst case {worst}")); + } + } +} + +#[derive(Default)] +struct OracleReport { + accepted: usize, + infeasible: Vec, + violations: Vec, +} + +/// Every shape alone, under every scope, and paired with a per-value +/// query on `m`'s readout, each with three random draws of copies: every +/// accepted plan's union-bound worst case is within δ = 0.01. Every shape +/// planned alone that has a plan without a failure probability has one +/// with it (Pass 2's level for a query meets (K) for that query). A +/// filter or `avg by (job)` above `m or b` has none either way: no error +/// contract covers its label over two sources. +#[test] +fn every_accepted_plan_meets_its_failure_probability_on_every_dataset() { + let mut rng = Rng(1); + let mut report = OracleReport::default(); + let scopes = [ + AccuracyScope::WholeResult, + AccuracyScope::PerValue, + AccuracyScope::GroupsWithRecordShareAtLeast(0.5), + ]; + let mut without_plan = Vec::new(); + for _ in 0..3 { + for &shape in SHAPES { + for scope in scopes { + let mut alone = OracleReport::default(); + check(&[(shape, scope)], Some(0.01), &mut rng, &mut alone); + if !alone.infeasible.is_empty() { + let mut without_delta = OracleReport::default(); + check(&[(shape, scope)], None, &mut rng, &mut without_delta); + if without_delta.infeasible.is_empty() { + without_plan.extend(alone.infeasible); + } + } + report.accepted += alone.accepted; + report.violations.extend(alone.violations); + } + check( + &[ + (RM, AccuracyScope::PerValue), + (shape, AccuracyScope::WholeResult), + ], + Some(0.01), + &mut rng, + &mut report, + ); + } + } + assert!(report.violations.is_empty(), "{:#?}", report.violations); + assert!(without_plan.is_empty(), "{without_plan:#?}"); + assert!( + report.accepted >= 3 * SHAPES.len() * scopes.len(), + "{}", + report.accepted + ); +} diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index f446486d1..da538707e 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeSet, HashMap}; use asap_cost::node::analytical::AnalyticalCostError; use asap_cost::{summary_type_name, CostModel}; -use asap_types::accuracy::AccuracyMetric; +use asap_types::accuracy::{AccuracyMetric, FailureProbabilityBound}; use asap_types::deployment_capabilities::ExclusionSource; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; @@ -223,20 +223,31 @@ pub enum ErrorSelectingPlan { /// Query `query`'s accuracy target is below the solver's precision: /// the MILP's selection misses it when checked exactly, so no plan is /// returned rather than one that may not meet the target. Example: a - /// failure probability of 1e-12 per output value (δ = 0.01 over 1e10 - /// values), below the solver's feasibility tolerance 1e-10. + /// failure probability of 1e-12 for each covered value (δ = 0.01 over + /// 1e10 values), below the solver's feasibility tolerance 1e-10. #[error( - "query {query}: an accuracy target is below the solver's precision (failure probability per output value {failure_probability_per_output_value:?}); the solver's plan misses it, so no plan is returned" + "query {query}: an accuracy target is below the solver's precision ({}); the solver's plan misses it, so no plan is returned", + failure_probability_bound_in_words(.failure_probability_bound) )] AccuracyTargetBelowSolverPrecision { query: usize, - failure_probability_per_output_value: Option, + failure_probability_bound: Option, }, /// The selection MILP implements cost model 1 only (#1095 §3). #[error("the selection MILP does not support {0}")] UnsupportedCostModel(String), } +/// `bound` in words for [`ErrorSelectingPlan::AccuracyTargetBelowSolverPrecision`]: +/// "failure probability: each of 100 covered values fails with at most +/// 0.0001", or "no failure probability bound" without one. +fn failure_probability_bound_in_words(bound: &Option) -> String { + match bound { + Some(bound) => format!("failure probability: {bound}"), + None => "no failure probability bound".to_string(), + } +} + /// The selection MILP is infeasible, with the reasons executions could not /// be costed. pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { diff --git a/crates/types/src/accuracy/mod.rs b/crates/types/src/accuracy/mod.rs index 67546063e..c0ae14bec 100644 --- a/crates/types/src/accuracy/mod.rs +++ b/crates/types/src/accuracy/mod.rs @@ -11,5 +11,6 @@ mod target; pub use default_metric::default_metric_for_root; pub use guarantee::{AccuracyGuarantee, BoundExpr, ProbabilityExpr}; pub use target::{ - AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, ReadoutValueCounts, + AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, FailureProbabilityBound, + NormScale, ReadoutValueCounts, }; diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs index 42efa60f2..85067ba33 100644 --- a/crates/types/src/accuracy/target/mod.rs +++ b/crates/types/src/accuracy/target/mod.rs @@ -167,9 +167,9 @@ impl AccuracyScope { /// when unbound: one for [`Self::PerValue`], `G` for /// [`Self::WholeResult`], and `min(⌊1 / share⌋, G)` for /// [`Self::GroupsWithRecordShareAtLeast`], which is bound even when `G` - /// is not. A target's failure probability splits over the readout values + /// is not. A target's failure probability is bounded over the estimates /// these values depend on - /// ([`AccuracyTarget::failure_probability_per_readout_value`]), so the + /// ([`AccuracyTarget::failure_probability_bound`]), so the /// strictest of several scopes is the one that covers the most values. pub fn max_values_covered(self, values_per_result: Option) -> Option { match self { @@ -298,30 +298,6 @@ impl AccuracyTarget { self.max_error == 0.0 && self.failure_probability == Some(0.0) } - /// The failure probability each output value of one evaluation must - /// meet: the target's δ split evenly, by the union bound, over the - /// values its scope covers when one evaluation returns - /// `values_per_evaluation` ($n_q$; [`AccuracyScope::max_values_covered`]). - /// `None` when the target sets no δ, and when that count is not known - /// for a δ over the whole result. A readout value's failure probability - /// splits δ over the readout values the covered output values depend - /// on instead ([`Self::failure_probability_per_readout_value`]). - /// - /// Example: δ = 0.01 over the whole result of a query returning 100 - /// series gives 0.0001 per output value; with the number of series - /// unknown, `None`; per value (`AccuracyScope::PerValue`), 0.01 either - /// way. - pub fn failure_probability_per_output_value( - &self, - values_per_evaluation: Option, - ) -> Option { - let delta = self.failure_probability?; - let values = self - .accuracy_scope - .max_values_covered(values_per_evaluation)?; - Some(delta / values.max(1) as f64) - } - /// The metrics this target accepts an error in: its metric, every /// metric when it names none. pub fn metrics(&self) -> Vec { @@ -331,17 +307,62 @@ impl AccuracyTarget { .collect() } - /// The failure probability each readout value below the query must - /// meet: δ split by the union bound over the readout values the output - /// values one statement covers depend on, `readout_values`, when one - /// evaluation returns `values_per_evaluation` ($n_q$). 1 when the - /// target sets no δ; `None` when no bound on that count is known. + /// Which union bound checks this target's failure probability, and + /// its limit, when one evaluation returns `values_per_evaluation` + /// ($n_q$) and the query reads `readout_values` (`R` and the fan-in). + /// `None` when the target sets no δ, and when neither bound is known. /// - /// The statement covers at most `V` output values - /// ([`AccuracyScope::max_values_covered`]), each depending on at most - /// `fan_in` readout values, and the query reads `R` in all, so it fails - /// only when one of at most `min(R, V · fan_in)` readout values does, - /// whatever their dependence. An unknown term drops out of the `min`. + /// The statement fails only when an estimate (one readout value in one + /// evaluation) a covered output value depends on fails, so its failure + /// probability is at most the summed δ of those estimates. Two sums + /// bound it: every estimate the query reads (at most `R`), or each of + /// the `V` covered values' own estimates (at most `V · fan-in`; + /// [`AccuracyScope::max_values_covered`]). The bound counting fewer + /// estimates is named; a tie, or `V · fan-in` unknown, names every + /// estimate read; `R` unknown names each covered value. + /// + /// Examples at δ = 0.01 over the whole result: + /// - `max(quantile_over_time(0.9, m[5m]))` over 1000 series: `R` 1000, + /// `V · fan-in` 1 · 1000, so every estimate read, at most 0.01. + /// - `q or r` over 100 series each, same labels: `R` 200, `V · fan-in` + /// 100 · 1, so each covered value, at most 0.01 / 100. + /// - `q * on() group_left e`, `e` one sketched series, 1000 values: + /// `R` 1, so every estimate read, at most 0.01. + pub fn failure_probability_bound( + &self, + values_per_evaluation: Option, + readout_values: ReadoutValueCounts, + ) -> Option { + let delta = self.failure_probability?; + let values_covered = self + .accuracy_scope + .max_values_covered(values_per_evaluation); + let estimates_of_covered_values = values_covered + .zip(readout_values.per_output_value) + .map(|(values, fan_in)| values.saturating_mul(fan_in)); + let every_estimate_read = FailureProbabilityBound::EveryEstimateRead { limit: delta }; + Some( + match (readout_values.per_evaluation, estimates_of_covered_values) { + (Some(read), Some(covered)) if read <= covered => every_estimate_read, + (Some(_), None) => every_estimate_read, + (_, Some(_)) => { + let values_covered = values_covered?.max(1); + FailureProbabilityBound::EachCoveredValue { + values_covered, + limit: delta / values_covered as f64, + } + } + (None, None) => return None, + }, + ) + } + + /// The failure probability each readout value below the query must + /// meet: the limit of [`Self::failure_probability_bound`] split evenly + /// over the estimates its sum counts, `R` for every estimate read and + /// the fan-in for each covered value. So a query whose readouts all + /// meet it meets its bound by construction. 1 when the target sets no + /// δ; `None` when no bound is known. /// /// Examples at δ = 0.01: /// - `quantile_over_time(0.99, m[5m])` over 8 series per value: @@ -355,19 +376,64 @@ impl AccuracyTarget { values_per_evaluation: Option, readout_values: ReadoutValueCounts, ) -> Option { - let Some(delta) = self.failure_probability else { + if self.failure_probability.is_none() { return Some(1.0); - }; - let readout_values_covered = self - .accuracy_scope - .max_values_covered(values_per_evaluation) - .zip(readout_values.per_output_value) - .map(|(values, fan_in)| values.saturating_mul(fan_in)); - let split_over = match (readout_values.per_evaluation, readout_values_covered) { - (Some(all), Some(covered)) => all.min(covered), - (all, covered) => all.or(covered)?, - }; - Some(delta / split_over.max(1) as f64) + } + Some( + match self.failure_probability_bound(values_per_evaluation, readout_values)? { + FailureProbabilityBound::EveryEstimateRead { limit } => { + limit / readout_values.per_evaluation?.max(1) as f64 + } + FailureProbabilityBound::EachCoveredValue { limit, .. } => { + limit / readout_values.per_output_value?.max(1) as f64 + } + }, + ) + } +} + +/// Which union bound a query's failure probability is checked by, with its +/// limit ([`AccuracyTarget::failure_probability_bound`]). Pass 2 splits δ +/// by it, and selection checks it on the readouts actually selected. +/// +/// Example: δ = 0.01 over the whole result of `q or r`, 100 series each: +/// `EachCoveredValue { values_covered: 100, limit: 1e-4 }`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum FailureProbabilityBound { + /// The summed δ of every estimate any output value depends on is at + /// most `limit`, the target's δ. + EveryEstimateRead { limit: f64 }, + /// The summed δ of the estimates one output value depends on is at + /// most `limit`, δ over the `values_covered` values the scope covers. + EachCoveredValue { values_covered: u64, limit: f64 }, +} + +/// The bound in words, for error messages: "every estimate read fails +/// with at most 0.01 in all" or "each of 100 covered values fails with at +/// most 0.0001". +impl std::fmt::Display for FailureProbabilityBound { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::EveryEstimateRead { limit } => { + write!(f, "every estimate read fails with at most {limit} in all") + } + Self::EachCoveredValue { + values_covered, + limit, + } => write!( + f, + "each of {values_covered} covered values fails with at most {limit}" + ), + } + } +} + +impl FailureProbabilityBound { + /// Its limit. + pub fn limit(&self) -> f64 { + match *self { + Self::EveryEstimateRead { limit } | Self::EachCoveredValue { limit, .. } => limit, + } } } diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs index 56b6af163..383c0a5fb 100644 --- a/crates/types/src/accuracy/target/tests.rs +++ b/crates/types/src/accuracy/target/tests.rs @@ -333,28 +333,166 @@ fn exact_targets() { ); } -/// A whole-result δ splits over the values one evaluation returns; a -/// per-value δ does not; no δ, or a whole-result one over an unknown count, -/// has none. +fn counts(per_evaluation: Option, per_output_value: Option) -> ReadoutValueCounts { + ReadoutValueCounts { + per_evaluation, + per_output_value, + } +} + +/// `q * on() group_left e`, `e` one sketched series matched onto 1000: +/// `R` = 1 is below `V · fan-in` = 1000, so every estimate read bounds the +/// whole result, at δ itself. #[test] -fn a_whole_result_failure_probability_splits_over_the_output_values() { +fn a_whole_result_reading_fewer_estimates_than_its_values_bounds_every_estimate_read() { let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + let bound = whole.failure_probability_bound(Some(1_000), counts(Some(1), Some(1))); assert_eq!( - whole.failure_probability_per_output_value(Some(100)), - Some(1e-4) + bound, + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) ); - assert_eq!(whole.failure_probability_per_output_value(None), None); - let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); assert_eq!( - per_value.failure_probability_per_output_value(None), + whole.failure_probability_per_readout_value(Some(1_000), counts(Some(1), Some(1))), Some(0.01) ); +} + +/// `q or r` over 100 series each: 200 estimates read, 100 · 1 a covered +/// value depends on, so each covered value is checked at δ / 100. +#[test] +fn an_or_over_more_estimates_than_its_values_bounds_each_covered_value() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + assert_eq!( + whole.failure_probability_bound(Some(100), counts(Some(200), Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 100, + limit: 1e-4 + }) + ); + assert_eq!( + whole.failure_probability_per_readout_value(Some(100), counts(Some(200), Some(1))), + Some(1e-4) + ); +} + +/// `max(q)` over 1000 series: every estimate read, 1e-5 per readout value. +#[test] +fn a_reduction_bounds_every_estimate_read() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + let read = counts(Some(1_000), Some(1_000)); + assert_eq!( + whole.failure_probability_bound(Some(1), read), + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) + ); + assert_eq!( + whole.failure_probability_per_readout_value(Some(1), read), + Some(1e-5) + ); +} + +/// A per-value target covers one value: δ is not divided by $n_q$. +#[test] +fn a_per_value_target_covers_one_value() { + let per_value = AccuracyTarget::max_error(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + assert_eq!( + per_value.failure_probability_bound(Some(8), counts(Some(8), Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 1, + limit: 0.01 + }) + ); +} + +/// At most ⌊1 / 0.05⌋ = 20 groups hold 5% of the records: over 1000 +/// estimates each covered group is checked at 0.01 / 20; over 10, every +/// estimate read at 0.01 (1e-3 per readout value). +#[test] +fn a_record_share_covers_at_most_one_over_its_share() { + let share = AccuracyTarget::max_error(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::GroupsWithRecordShareAtLeast(0.05)); + assert_eq!( + share.failure_probability_bound(Some(1_000), counts(Some(1_000), Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 20, + limit: 5e-4 + }) + ); + assert_eq!( + share.failure_probability_bound(Some(1_000), counts(Some(10), Some(1))), + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) + ); + assert_eq!( + share.failure_probability_per_readout_value(Some(1_000), counts(Some(10), Some(1))), + Some(1e-3) + ); +} + +/// An unknown count leaves the other bound; with both unknown there is +/// none, and no target δ gives none either. +#[test] +fn an_unbound_count_leaves_the_other_bound() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + assert_eq!( + whole.failure_probability_bound(None, counts(Some(1_000), Some(1))), + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) + ); + assert_eq!( + whole.failure_probability_bound(Some(100), counts(None, Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 100, + limit: 1e-4 + }) + ); + assert_eq!( + whole.failure_probability_bound(None, counts(None, Some(1))), + None + ); assert_eq!( - AccuracyTarget::max_error(0.01).failure_probability_per_output_value(Some(100)), + AccuracyTarget::max_error(0.01) + .failure_probability_bound(Some(1), counts(Some(1), Some(1))), None ); } +/// Pass 2's per-readout split times the estimates the named bound counts +/// (`R`, or `V · fan-in`) is δ: readouts at the split meet the bound by +/// construction. +#[test] +fn the_readout_split_meets_the_bound_it_names() { + for delta in [1e-2, 1e-6] { + for values in [Some(1), Some(7), Some(100), None] { + for read in [Some(1), Some(7), Some(200), None] { + for fan_in in [Some(1), Some(7), read] { + let target = AccuracyTarget::max_error(0.01).failure_probability(delta); + let readout_values = counts(read, fan_in); + let Some(bound) = target.failure_probability_bound(values, readout_values) + else { + continue; + }; + let split = target + .failure_probability_per_readout_value(values, readout_values) + .unwrap(); + let total = match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => { + split * read.unwrap().max(1) as f64 + } + FailureProbabilityBound::EachCoveredValue { values_covered, .. } => { + split * (values_covered * fan_in.unwrap().max(1)) as f64 + } + }; + assert!( + (total - delta).abs() <= 1e-12 * delta, + "{values:?} {readout_values:?}: {total}" + ); + } + } + } + } +} + /// δ = 0.01 splits over min(R, V · fan-in) readout values; an unknown term /// drops out of the `min`. #[test] @@ -422,3 +560,20 @@ fn readout_value_counts_combine_count_by_count() { counts(None, Some(2)) ); } + +/// A bound reads as words in error messages, not as its `Debug` form. +#[test] +fn a_failure_probability_bound_reads_as_words() { + assert_eq!( + FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }.to_string(), + "every estimate read fails with at most 0.01 in all" + ); + assert_eq!( + FailureProbabilityBound::EachCoveredValue { + values_covered: 100, + limit: 1e-4 + } + .to_string(), + "each of 100 covered values fails with at most 0.0001" + ); +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 2a108ebca..020177575 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -178,6 +178,39 @@ impl Memo { smaller_known(by_keys, by_rows) } + /// The most estimates (sketch readout values) one evaluation of the + /// readout group `group` computes: its values + /// ([`Self::max_values_per_evaluation`]), except for a top-k, which + /// estimates every distinct item of each state it ranks. That is at + /// most its states' count times the distinct items one state counts + /// ([`Self::max_distinct_per_evaluation`]), and at most the rows (or + /// series) its values are bounded by, as each item comes from a row; + /// the smaller known one. This is the `R` a readout's failure + /// probability is charged for when an output reads all its values. + /// The distinct-item bound is the deployment's contract, as HLL's + /// error is: a source declaring fewer distinct items than a state + /// counts gives too small an `R`, and a δ split over too few + /// estimates. + /// + /// Example: `topk(10, sum by (dst) (count_over_time(flows[5m])))` as + /// one heap per `job`, with 3 jobs and at most 500 distinct `dst` per + /// evaluation declared, estimates at most 3 × 500 = 1500 items, while + /// it returns 30 values. + pub fn max_estimates_per_evaluation(&self, group: MemoGroupId) -> Option { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query: SketchStatistic::TopK { .. }, + }) = &self.groups[group.0 as usize].expression + else { + return self.max_values_per_evaluation(group); + }; + let by_distinct_items = self + .max_values_per_evaluation(*summary_input) + .zip(self.max_distinct_per_evaluation(group)) + .and_then(|(states, items)| states.checked_mul(items)); + smaller_known(by_distinct_items, self.max_values_per_evaluation(group)) + } + /// The most rows one evaluation of `group` returns, from the facts the /// data workload declares; `None` when they bound it in no way. A `Scan` /// returns the rows its source holds (`SourceFacts::rows_held`), and a diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 1ba98db1d..9a11d59d2 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -464,6 +464,80 @@ fn max_values_per_evaluation_bounds_a_readout_by_its_states_keys() { } } +/// A top-k readout estimates every distinct item of each state it ranks: +/// over `latency` with 1000 series, 20 jobs and at most 30 distinct items +/// per evaluation declared, a heap per job estimates at most 20 × 30 = +/// 600 items (fewer than the 1000 series its rows come from), and one +/// heap over everything 30. Without the distinct bound, the series bound +/// it: 1000. A quantile estimates its values, one per state. +#[test] +fn max_estimates_per_evaluation_counts_every_item_a_top_k_ranks() { + for (max_distinct, reduction_is_by_job, estimates) in [ + (Some(30), true, 600), + (Some(30), false, 30), + (None, true, 1000), + ] { + let data = crate::workload::DataWorkload { + sources: [( + "latency".to_string(), + crate::workload::SourceFacts { + series_count: Some(1000), + distinct_values_per_label: [("job".to_string(), 20)].into(), + max_distinct_per_evaluation: max_distinct, + ..Default::default() + }, + )] + .into(), + ..Default::default() + }; + let mut memo = Memo::new(&data); + let input = memo + .find_or_add_group(scan( + "latency", + &[("job", crate::ir::schema::DataType::Utf8)], + vec![], + )) + .unwrap(); + let job = memo.groups()[input.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == "job") + .expect("the scan lists job"); + let reduction = if reduction_is_by_job { + Reduction::by(vec![job]) + } else { + Reduction::by(vec![]) + }; + let state_of = |memo: &mut Memo, algorithm| { + memo.find_or_add_group(state_grouped_by( + FieldDataType::Sketch(algorithm), + reduction.clone(), + input, + )) + .unwrap() + }; + let heap = state_of(&mut memo, SketchAlgorithm::CmsWithHeap); + let top_k = memo + .find_or_add_group(readout(SketchStatistic::TopK { k: 5 }, heap)) + .unwrap(); + assert_eq!( + memo.max_estimates_per_evaluation(top_k), + Some(estimates), + "{max_distinct:?} {reduction_is_by_job}" + ); + let kll = state_of(&mut memo, SketchAlgorithm::Kll); + let quantile = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.9 }, kll)) + .unwrap(); + assert_eq!( + memo.max_estimates_per_evaluation(quantile), + memo.max_values_per_evaluation(quantile) + ); + } +} + /// A distinct count over `events`, bounded at 9,636 distinct items per /// evaluation, counts at most 9,636 in every group above the scan; a join /// of two bounded sources counts at most their sum; a source without a diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 34711cabc..e430af641 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -4,6 +4,7 @@ use std::collections::BTreeSet; +use crate::accuracy::{FailureProbabilityBound, ReadoutValueCounts}; use crate::stages::{ CandidateForMemoGroupId, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate, RootPruneReason, @@ -66,9 +67,39 @@ pub struct LogicalASAPMemoCandidates { /// (its MILP's constraint (X)); Pass 2 only lists it. Empty for a query without an accuracy /// target. pub exact_only_groups_of_query: Vec>, + /// Indexed by query: the readout values (estimates) its output can + /// depend on, `R` per evaluation and the fan-in per output value, as + /// Pass 2 counted them to split the query's δ over its readouts; + /// `None` for a query without an accuracy target. Stage 3's constraint + /// (K) checks the bound they name + /// ([`Self::failure_probability_bound`]), so the split and the check + /// cannot choose differently. Example: `max(quantile_over_time(0.9, + /// m[5m]))` over 1000 series: `R` 1000, fan-in 1000. + pub readout_values_of_query: Vec>, } impl LogicalASAPMemoCandidates { + /// The union bound query `query`'s failure probability is checked by, + /// with its limit: its target's + /// (`AccuracyTarget::failure_probability_bound`) over the readout + /// values Pass 2 counted for it ([`Self::readout_values_of_query`]). + /// `None` without a target δ, or with neither bound known (the query + /// then reads its readouts only exactly). + /// + /// Example: δ = 0.01 over the whole result of + /// `max(quantile_over_time(0.9, m[5m]))` over 1000 series: every + /// estimate read, at most 0.01. + pub fn failure_probability_bound(&self, query: usize) -> Option { + let memo_query = &self.memo.queries()[query]; + memo_query + .accuracy_target + .as_ref()? + .failure_probability_bound( + memo_query.number_of_values_per_query_evaluation, + self.readout_values_of_query.get(query).copied().flatten()?, + ) + } + /// Whether query `query` must not read `candidate`: it is not exact /// (it has sketch parameters θ) and its group is one the query may read /// only exactly ([`Self::exact_only_groups_of_query`]). A readout's own diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 622211238..1af97abe8 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -133,6 +133,7 @@ fn candidates( pruned: vec![], evaluation_grid_of_group, exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, vec![], vec![], @@ -692,6 +693,7 @@ fn new_indexes_only_admissible_edges() { pruned: vec![], evaluation_grid_of_group: vec![None; 5], exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, physical_candidates, edges.clone(), @@ -783,6 +785,7 @@ fn a_stored_chain_and_an_ingestion_time_result_are_stored_on_their_refresh_windo pruned: vec![], evaluation_grid_of_group: vec![grid_60s; group_count], exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, vec![ candidate(series, timing, Lifetime::None), diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index b4a8bc108..e354e511d 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -79,7 +79,9 @@ heap size and its value guarantee alone (the contracts table). Pass 2 gives each sketch readout the error and per-value failure probability `δ / min(R, V · fan_in)` it must meet -(`AccuracyTarget::failure_probability_per_readout_value`): the statement +(`AccuracyTarget::failure_probability_per_readout_value`, the limit of the +bound `AccuracyTarget::failure_probability_bound` names split over the +estimates that bound counts): the statement fails only when a readout value one of its `V` covered output values depends on does. `R` is the readout values the query reads per evaluation (the values per evaluation of every readout group below it, summed), and `fan_in` @@ -89,7 +91,11 @@ for a global `max`; for `sum by (job)` too, since the declared facts bound how many jobs there are, not how many series one job holds), the sum over the sides of a binary operator, the larger side of `or` or input of a concatenation. A top-k readout chooses its output keys by their estimates, -so each of its output values depends on every value it ranks. A readout +so each of its output values depends on every item it ranks, and its `R` +counts those items, not its output values +(`Memo::max_estimates_per_evaluation`): at most its states' count times the +distinct items one state counts (`SourceFacts::max_distinct_per_evaluation`), +and at most the rows its values come from. A readout group whose candidates include a reduction over readouts below it takes the larger of the two counts. Both counts are per evaluation, not per subquery step. An unknown term drops out of the `min`; with both unknown no per-value @@ -106,7 +112,14 @@ one: DDSketch's relative error does not depend on δ, so of (5%, 0.01) and build per listed θ, and each edge checks the guarantee of its producer's θ at the required failure probability, so a build several readers share serves the readers whose levels its θ meets. Stage 3 only checks the guarantee; it -never resizes. At equal δ, the scope covering the most values is the +never resizes. Stage 3's constraint (K) bounds the same estimates with the +same counts (`R` and the fan-in, from one walk, +`asap_accuracy::readout_value_counts_of_queries`, and one test of whether an +input's error reaches the output), so for each query planned alone a +readout at this split meets (K). A level another query gave is checked, not +trusted ("Which estimates a result depends on", below). When two queries +share an execution and need incompatible levels, no level is given for the +pair, and the selection may find no sketch plan for one of them. At equal δ, the scope covering the most values is the strictest: every group (`G`, when `G` is above 100), then a 1% share (100), then a 5% share (20), then a single value (1). A whole-result target with an unbound `G` splits over `R`, so `R` must be an upper bound. A readout @@ -300,17 +313,24 @@ has no value bound, so neither passes through division. Likewise, an exact sum above approximate values retains their uncertainty: exact arithmetic does not recover information lost below it. -Probability is not propagated through operators. A query's failure -probability is split into a per-value one for each readout; a composed query -whose readouts' failure events add up (a ratio of two quantiles) reads each at -a level of a failure-probability grid, and the levels sum within the query's -per-value failure probability by the union bound, without assuming -independence between child errors. Estimator-local assumptions, such as -independent bucket hashing, must be stated separately. Shared DAG nodes do -not create independent errors merely because several consumers reference -them, and sharing does not justify reducing the failure budget. Per-result -guarantees also do not imply a simultaneous guarantee across every query or -evaluation time in a dashboard. +Failure probability is not a contract factor: an operator does not scale +it. What fails is an estimate, one value a sketch readout returns in one +evaluation (one series' quantile, one group's distinct count, one key's +frequency), which misses its error bound with probability at most its +readout's δ. A query's result fails only when an estimate it depends on +does, so its failure probability is at most the sum of δ over the estimates +it can depend on (the union bound, without assuming independence between +estimates). Pass 2 splits a query's δ over those estimates; selection checks +the sum for the readouts actually selected ("Which estimates a result +depends on"). A composed query (a ratio of two quantiles) reads each readout +at a level of a failure-probability grid and the selection combines them. +Estimator-local assumptions, such as independent bucket hashing, must be +stated separately. Shared DAG nodes do not create independent errors merely +because several consumers reference them. Sharing never earns a larger δ for +an estimate: an estimate several output values read is one event, which is +why it is counted once per path, not once per reader. Per-result guarantees +also do not imply a simultaneous guarantee across every query or evaluation +time in a dashboard. ### Sign conditions @@ -425,9 +445,24 @@ each query's target. This is constraint (K). **Variables.** Per execution `c` (a candidate in one evaluation context) that a query with a target reaches, and that is its own original (not in a -copy of an equivalent context): its error bound `E_{c,m}` per metric `m` and -its failure probability `D_c`, continuous; per input `i`, `F_{c,i,m}` and -`H_{c,i}`, what it reads from the producer selected on `i`. +copy of an equivalent context), all continuous: +- its error bound `E_{c,m}` per metric `m`; +- `P_c`, the failure probability of one of its output values: the largest, + over its output values, of the summed δ of the estimates that value + depends on; +- `W_c`, the failure probability of its whole output: the summed δ of every + estimate any of its output values depends on. It exists only where every + readout below `c` bounds its estimates per evaluation (`R` known). + +Each exists only where some selection makes it positive and its error is +not constant. Per input `i`, `F_{c,i,m}`, `HP_{c,i}` and `HW_{c,i}` are what +`c` reads from the producer selected on `i`. `P` and `W` exist only where a +query root above checks them, through the rows that read them: a root +checking each covered value reads `P`, one checking every estimate read +`W`; `P` reads `P` of an input read value by value and `W` of one read +whole; `W` reads `W` (`ErrorModel::checked_failure_probabilities`). With +both created wherever bounded, RangeHeavy-8users solved about 3× slower +with the same plan. **Rows.** With `y_{u,c,i}` the edge variables and `z_{q,v}` the choice of execution `v` answering query `q`: @@ -436,45 +471,196 @@ execution `v` answering query `q`: `F_{c,i,m} ≥ E_{u,m} − M_{u,m}(1 − y_{u,c,i})` per selectable producer `u`; `ε_c` is a readout's own error, its required accuracy (`CandidateForMemoGroup::required_accuracy`), 0 for any other candidate. + An operator that copies each output value from one input (`or`, a + concatenation) has one row per input instead of the sum: its output's + error is one input's, `E_{c,m} ≥ ε_c[m = m_c] + β_{c,i,m} F_{c,i,m}` for + each `i`. - The metric rule on the selected path: `E_{u,m} ≤ M_{u,m}(1 − y_{u,c,i})` when `c` has no contract in `m`. An input with `β = 0` passes none. - The validity cap: `F_{c,i,m} ≤ b_{c,i}` where the contract's linear bound holds only up to `b` (a ratio's denominator). -- `D_c ≥ δ_c + Σ_i H_{c,i}`, `H_{c,i} ≥ D_u − M_u(1 − y_{u,c,i})`: a union - bound over the producers selected. -- At each output of a query with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` +- Failure probabilities, with `HP_{c,i} ≥ P_u − M^P_u(1 − y_{u,c,i})` and + `HW_{c,i} ≥ W_u − M^W_u(1 − y_{u,c,i})` per selectable producer `u`, over + only the inputs whose estimates reach the output (`β > 0` in a metric of + `c`'s contract, or values chosen by estimates, the test Pass 2's split + uses; a count's input and the right input of `and` and `unless` add + nothing unless a top-k chooses their values): + - a readout: `P_c = δ_c` and `W_c = R_c δ_c`, with `δ_c` its required + failure probability and `R_c` its estimates per evaluation + (`Memo::max_estimates_per_evaluation`); a top-k readout, whose output + keys are chosen by every estimate it ranks, has `P_c = W_c = R_c δ_c`. + Constants, as a readout's error is; + - any other execution: + `P_c ≥ Σ_{i read value by value} HP_{c,i} + Σ_{i read whole} HW_{c,i}`, + where an operator each of whose output values depends on one input's + estimates (`or`, a concatenation, with no top-k choosing an input's + values) has instead `P_c ≥ HP_{c,i} + Σ_{i read whole} HW_{c,i}` for + each input `i` read value by value; and `W_c ≥ Σ_i HW_{c,i}`. The + operator classes are in the table below. +- At the outputs of a query `q` with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` in each metric the root allows (the target's at an operator root, none for - the operands of a scalar expression no contract covers, 0 elsewhere), and - `D_v ≤ δ_q / V + M(1 − z_{q,v})` with `V` the values the target's scope - covers (`AccuracyTarget::failure_probability_per_output_value`, `n_q` for a - whole result); no `D` row without a δ. Each limit is widened as the edge - check widens it (`tolerated_limit`). - -`D` adds each selected producer's δ once, not once per readout value an -output value depends on: `max(q)` over 1,000 series reads 1,000 readout -values. That is sound because Pass 2's levels already split the query's δ -over the readout values its output can depend on, so each readout's δ is the -query's over 1,000. - -**Constant errors.** A readout's state is exact, so its error is its own -requirement under every selection: it has no `E` or `D` variable, and its -consumers read `Σ_u ε_u y_{u,c,i}` (at most one `y` of an input is 1). This is + the operands of a scalar expression no contract covers, 0 elsewhere). With + a δ, one failure row per query, over every output `v` it may be answered + from, by the bound `AccuracyTarget::failure_probability_bound` names from + `V` (the values the target's scope covers) and the query's counts `R` and + fan-in (Pass 2's `ReadoutValueCounts`). Pass 2 records the counts it split + δ by (`LogicalASAPMemoCandidates::readout_values_of_query`), and (K) + checks the bound they name: + - every estimate read, when `R ≤ V · fan_in` or `V` is unbound: + `W_v ≤ δ_q + M(1 − z_{q,v})`; + - each covered value, when `V · fan_in < R` or `R` is unbound: + `P_v ≤ δ_q / V + M(1 − z_{q,v})`; + - neither, when both are unbound: no row. Pass 2 gave the query no level, + and constraint (X) makes it read the readout groups below it only + exactly; that, not (K), is what keeps it from reading a level another + query gave. + + An output whose checked term is unbounded under some selection (a readout + below with unknown `R`) is not answered by that execution. A query with + several roots (a scalar expression over them) sums its roots' terms in its + one row. Each limit is widened as the edge check widens it + (`tolerated_limit`). + +**Which estimates a result depends on.** The union bound is taken over +estimates, not over readouts or output values. + +*Soundness.* Every operator above a readout is deterministic, and its error +contract holds whenever its inputs are within their bounds. So an output +value misses `ε_q` only if an estimate it depends on misses its bound. Which +estimates an output value depends on is decided by the data's series and +keys (labels, group keys, join keys), not by the estimates. The exceptions +are operators that choose by estimated value: a top-k readout is charged for +every value it ranks, and any other such operator (a comparison, a `LIMIT` +after ordering by an estimate) has no contract. An operator keeping or +taking values by the keys a top-k chose (`p and topk(3, q)`, `p unless +topk(3, q)`, `topk(3, q) or p`, also through operators reading the top-k +value by value) depends on every estimate the top-k ranks: that input's +estimates count even where `β = 0`, and `or` sums its sides' `P` +(`asap_accuracy::values_chosen_by_estimate`). So for any data `D` the +statement fails only on `⋃_{e ∈ S_q(D)} E_e`, where `S_q(D)` is the +estimates the covered output values depend on and `E_e` is estimate `e`'s +miss. Hence + + P(fail) ≤ max_D Σ_{e ∈ S_q(D)} δ_e. + +Both root rows bound the right side for every `D` the declared facts allow: +- every estimate read: `S_q(D)` lies in the estimates of the readouts below, + at most `R_c` per readout per path, so the sum is at most `W_v`; +- each covered value: `S_q(D) = ⋃_{v covered} deps(v)`, at most `V` values, + each with `Σ_{e ∈ deps(v)} δ_e ≤ P_v`, so the sum is at most `V · P_v`. + +The recursions are sound step by step: +- a value read value by value is one input value, whose estimates sum to at + most `P_u`; +- a value copied from one input (`or`) is that input's value, so the largest + of the inputs' `P` bounds it; +- an input read whole (a reduction) may contribute every estimate of that + input, at most `W_u`; +- a whole output depends on at most the union of its inputs' estimates, at + most `Σ W_u`. + +An exact input has δ = 0. An input with `β = 0` contributes only which series +or keys it has. A readout returns exactly the series the exact evaluation +does, so those estimates cannot make the output miss, except a top-k's, +whose keys are chosen by its estimates (above). + +*The assumption it rests on.* Soundness needs `R_c` to be an upper bound on +the estimates one evaluation of readout `c` computes, under every dataset +the declared facts allow; it is the only declared count the bound trusts +(`n_q` only decides which row is checked). `R_c` is +`Memo::max_estimates_per_evaluation`: for a readout other than a top-k its +values, `Memo::max_values_per_evaluation`, the product of the distinct +values its state's keys' scans declare, at most the series for PromQL +labels, clamped to the rows its input reads. The declared distinct items +are the deployment's contract, as HLL's are: declaring fewer than a state +counts gives a too small `R_c`. Two cases checked against it: +- A top-k readout ranks every distinct item of each state, more than the + values it returns. Its `R_c` is the smaller of its states' count times the + declared distinct items per state (`SourceFacts::max_distinct_per_evaluation`) + and the series or rows its items come from, both upper bounds on the items + (`Memo::max_estimates_per_evaluation`). +- A time-bucketed SQL grouping (`GROUP BY src, date_trunc('minute', ts)`) + returns buckets × keys values. A time or computed key bounds nothing by + itself, so `R_c` falls back to the rows the grouping reads, which bounds + the groups whatever the buckets. The executed bounds check + (`memo_value_bounds`) runs these shapes on data the facts allow. + +*Scopes.* `PerValue` covers one value: `V = 1`, so the row is `P_v ≤ δ`, with +no division by `n_q`. `WholeResult` covers `V = n_q`. +`GroupsWithRecordShareAtLeast(s)` covers `V = min(⌊1/s⌋, G)` groups. Which +groups those are is decided by exact record counts, not by estimates, so the +same two rows hold with that `V`, and `W_v` (all groups) stays a sound +alternative. + +*Operator classes* (shared with Pass 2's split: +`asap_accuracy::inputs_read_value_by_value`, +`ErrorMetricsOfMemo::each_value_depends_on_one_input` and the "estimates +reach the output" test `ErrorMetricsOfMemo::input_estimates_reach_output`): + +| Class | Operators | `P_c` | `W_c` | +|---|---|---|---| +| Readout | `SummaryEstimate` (not top-k) | `δ_c` | `R_c δ_c` | +| Readout choosing by estimate | top-k | `R_c δ_c` | `R_c δ_c` | +| Value by value | filter on copied columns, project, sort, time shift, relabel, label enrichment, series sample; binary arithmetic, one-to-one or many-to-one (`on`, `ignoring`, `group_left`, `group_right`: each output value reads one value of each side, and the "one" side's value is read by many outputs but is one estimate); join | `Σ_i HP` | `Σ_i HW` | +| Each value from one input | `or`, concatenation (`UNION ALL`); `and`, `unless` (left only, the right has `β = 0`) | `max_i HP`; `Σ_i HP` when a top-k chooses an input's values | `Σ_i HW` | +| Input read whole | aggregates (`sum`/`max`/`avg` by or without keys), any operator whose output value may read many values of an input, the operand of `scalar` | `Σ_i HW` | `Σ_i HW` | +| No contract | a subquery, a comparison, a counter rate | none: no error, and so no estimate, passes | none | +| `β = 0` input | a count's input, the right of `and` and `unless` | adds 0, or as read value by value or whole when a top-k chooses its values | adds 0, or `HW` when a top-k chooses its values | + +*Same rule as Pass 2, for a query planned alone.* Pass 2 gives each readout +of a query one level `δ / min(R, V · fan_in)`. Its `R` and fan-in add, per +path, the terms `W` and `P` add, and take the largest over a group's +candidates where the rows take the selected one. So at that level, +`W_v ≤ R · δ / min(…)` and `V · P_v ≤ V · fan_in · δ / min(…)`. The row the +root checks is the one whose count is the `min`, so it is at most δ. A level +a stricter rule gave (any `δ_c ≤ δ / max(V, R)`) meets it too. A level given +for another query with a looser split does not: `max(q)` over 1,000 series +cannot read a copy at the 0.01 a per-value query on `q` was given, because +`W = 1,000 × 0.01 > 0.01`. + +*What the rows lose* (all sound; each only asks for a stricter δ than +needed): +1. An estimate one query reaches on several paths counts once per path: + `max(q) − min(q)` over 1,000 series charges `2,000 δ_c`, although 1,000 + estimates fail at most. Pass 2's `R` counts it the same way, so no level + it gives becomes infeasible. +2. The root checks one of the two rows, chosen from the counts. When the + readouts below have different δs (a composed query), the other row may + have been smaller. +3. An operator that returns fewer values than it reads does not lower the + count of a whole input (a label filter above a shared readout, a join + dropping unmatched rows), as in Pass 2's `R`. + +Counting each estimate exactly once, per query, needs a variable per +(query, execution) on the query's selected path, and at each `or` a dual +variable bounding how many output values each side supplies. That is linear +but grows with queries × executions; it is not used. + +**Constant errors.** A readout's state is exact, so its error and failure +probabilities are its own under every selection: it has no `E`, `P` or `W` +variable, and its consumers read `Σ_u ε_u y_{u,c,i}`, `Σ_u δ_u y_{u,c,i}` and +`Σ_u R_u δ_u y_{u,c,i}` (at most one `y` of an input is 1). This is exactly the variable form with `E_u = ε_u`, and it removes the big-M rows of every readout. -**Ceilings.** `M` is the largest error any selection gives an execution, +**Ceilings.** `M` (and `M^P`, `M^W`) is the largest error and failure +probability any selection gives an execution (unbounded for `P` or `W` when +a readout below has unknown `R`; then the execution has no such variable), computed in one pass in the memo's topological order (every selectable edge reads a group ordered before the consumer's), each input taking its largest producer's ceiling, by the same composition rule as the rows. The rule is -written once (`AccuracyRules::compose`), and also checks a whole selection -(`AccuracyRules::selection_meets_targets`): the greedy upper bound that prunes +written once (`AccuracyRules::compose`, composing `E`, `P` and `W`), and +also checks a whole selection (`AccuracyRules::selection_meets_targets`, the +same root row): the greedy upper bound that prunes executions dearer than a feasible plan uses its plan only when it meets (K), since the cheapest readouts of a ratio may each meet their level and compose above the target. **Solver tolerance.** HiGHS's feasibility tolerances are absolute, 1e-7 by -default, while a per-value δ is often far smaller: δ = 0.01 over a whole -result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth: two +default. A row checking every estimate read compares sums of the size of δ +itself (0.01 against 0.01), where 1e-7 is negligible. A row checking each +covered value compares `P_v` with δ / V, often far smaller: δ = 0.01 over a +whole result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth: two readouts failing with 5.4e-7 each compose to 1.08e-6 and pass that limit within the default tolerance. So HiGHS first solves at its defaults, and the selection is then checked exactly (`AccuracyRules::selection_meets_targets`, @@ -484,20 +670,26 @@ A row may still pass its limit by up to 1e-10, which for a per-value δ near or below 1e-10 can be orders of magnitude: at δ = 1e-12 per value (δ = 0.01 over 1e10 values) the solver may select readouts failing with 1.08e-12 in all. A selection that misses a target after the second solve is an error -naming the query and its per-value δ ("an accuracy target is below the -solver's precision"); planning never returns a plan that may not meet its +naming the query and its bound with its limit ("an accuracy target is below +the solver's precision"); planning never returns a plan that may not meet its target. The stricter tolerance is paid for only when the check needs it: solving every (K) at 1e-10 made RangeHeavy-8users about 6× slower, deriving the tolerance from the smallest limit solved RollupHeavy-8users about 2.8× slower, and scaling (K)'s rows by their limits solved RangeHeavy-8users -about 4× slower. +about 4× slower, and dividing every failure probability by the smallest +root limit (so the tightest limit is 1) solved RangeHeavy-8users 10% and +RangeHeavy-8users-counter 48% slower. **Context copies.** A copy of an equivalent evaluation context shares its original's variables and has no rows of its own. Two contexts are copies only -when each output's root limits (the tolerated error per metric and failure -probability per output value) are equal, so the copy's rows would repeat the -original's: queries at δ = 0.01 per value returning 10 and 100 values are -copies; over a whole result they are not. +when each output's root limits (the tolerated error per metric, and the +failure probability bound's kind and limit, all the rows read of it) are +equal, so the copy's rows would repeat the original's: +- queries at δ = 0.01 per value returning 10 and 100 values are copies; +- over a whole result they are copies when both check every estimate read + (a readout of 10 series declaring 10 and 50 values); +- they are not copies when they check each covered value of 10 and of 100 + values. ### Example: allocation in a ratio @@ -517,6 +709,30 @@ levels of the grid up to its cap; with levels 0.02 (cost 1) and 0.01 (cost 3): (K) selects both 0.01 readouts, cost 6. A greedy plan reading the two cheapest readouts (cost 2) misses B's target, so it bounds nothing. +### Example: failure probability in `or`, many-to-one matches and reductions + +All at δ = 0.01 over the whole result unless stated. + +| Query | Facts | Pass 2's level (`δ / min(R, V · fan_in)`) | Row (K) checks | Value | Estimates that can fail at once | +|---|---|---|---|---|---| +| `q or r` | 100 series each, same labels, `n_q` = 100 | `R` = 200, fan-in 1: `0.01 / min(200, 100) = 1e-4` | each covered value: `max(1e-4, 1e-4) ≤ 0.01 / 100` | 1e-4 ✓ | 100 (each output copies one) | +| `q * on() group_left e` | `q` exact over 1,000 series, `e` one sketched series, `n_q` = 1,000 | `R` = 1, fan-in 1: `0.01 / min(1, 1,000) = 0.01` | every estimate read: `W = 0 + 1 · 0.01` | 0.01 ✓ | 1 (every output reads `e`'s one estimate) | +| `max(q)` | 1,000 series, `n_q` = 1 | `R` = 1,000, fan-in 1,000: `1e-5` | every estimate read: `1,000 · 1e-5` | 0.01 ✓ | 1,000 | +| `count(distinct user_id)` by service, per value | δ = 0.05, ε = 0.05 relative, 10,000 services | `R` = 10,000, `V` = 1, fan-in 1: `0.05 / 1 = 0.05` | each covered value, `V` = 1: `P = 0.05` | 0.05 ✓ | 1 per value | +| `A or B`, `A` = `quantile_over_time(0.99, m{env="prod"}[5m])`, `B` = `quantile_over_time(0.99, m[5m])`, both reading one readout of `m` | 100 series, `n_q` = 100 | `R` = 200 (one readout, two paths), fan-in 1: `1e-4` | each covered value: `max(1e-4, 1e-4) ≤ 0.01 / 100` | 1e-4 ✓ | 100 | +| `max by (job) (A or B)` | as above, 10 jobs | `R` = 200, fan-in 200: `5e-5` | every estimate read: `W = 200 · 5e-5` | 0.01 ✓ | 100: the rows charge each estimate twice (once per path), so 1e-4 would have sufficed | + +Charging the selected readout's δ once against δ / `n_q` would instead: +- reject `q or r` (`2e-4 > 1e-4`) and the many-to-one match + (`0.01 > 0.01 / 1,000`), leaving them no sketch plan; +- accept for `max(q)` any copy of the readout, including one a per-value + query on `q` was given at 0.01, whose 1,000 estimates may fail with up to + 1,000 × 0.01. + +For the distinct count, a split over `max(V, R)` would ask each group's KMV +for 0.05 / 10,000 = 5e-6, k = 26,656. The per-value row asks only that each +group's own estimate fail with at most 0.05, which KMV meets at k = 4,323. + ## Evidence and trust boundaries `AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the