From 40664b5f90689d6ae30690b5b326fad167e1021c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:59:03 +0000 Subject: [PATCH 1/5] test(stage3): a reduction must not read a readout level given for a per-value query max(q) over 1,000 series at delta 0.01 reads a copy failing with 5e-4 that a per-value query on q at delta 0.5 was given; its union bound is 0.5. Fails on main: (K) adds the selected readout's delta once. --- .../src/milp/model/tests.rs | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 59dfdc616..038e0d5bb 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeSet, HashMap}; use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::operator::Reduction; +use asap_types::ir::operator::{AggIntent, Reduction}; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::ColumnRef; @@ -25,7 +25,7 @@ use asap_types::summary_type::{ }; use asap_types::test_support::memo_builders::{ group, own_candidate, quantile_over_time_readouts, query, scan_group, sketch_state, - try_candidate, + try_candidate, with_series, }; use asap_types::test_support::memo_fixtures; use asap_types::test_support::memo_fixtures::arithmetic; @@ -1517,6 +1517,55 @@ fn contexts_whose_queries_split_their_failure_probability_differently_are_not_co ); } +/// `q` = p90 over 1,000 series, read per value at δ = 0.5, and `max(q)` +/// over the whole result at δ = 0.01, with one readout group holding a +/// copy failing with 5e-4 (cost 1, the level `q` alone needs) and one +/// failing with 1e-5 (cost 3). `max(q)` may fail when any of the 1,000 +/// estimates it reads does, so the 5e-4 copy gives it a failure +/// probability up to 1,000 × 5e-4 = 0.5, not 0.01: only the 1e-5 copy +/// (1,000 × 1e-5 = 0.01) meets its target. A rule adding the selected +/// readout's δ once accepts the 5e-4 copy for both queries. +#[test] +fn k_does_not_let_a_reduction_read_a_level_given_for_a_per_value_query() { + let mut memo = Memo::new(&with_series("m", 1_000)); + let (readout_group, readouts) = + ddsketch_readouts(&mut memo, 0.9, &[(0.01, 5e-4), (0.01, 1e-5)]); + let max = group( + &mut memo, + memo_fixtures::aggregate( + Reduction::by(vec![]), + AggIntent::Max { col: None }, + readout_group, + ), + ); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some( + relative(0.01) + .failure_probability(0.5) + .accuracy_scope(AccuracyScope::PerValue), + ), + number_of_values_per_query_evaluation: Some(1_000), + }); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(max), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + // The scan costs 10, so neither readout is dearer than a feasible plan. + let mut w = vec![0.0; physical.physical_candidates.len()]; + w[0] = 10.0; + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + let (solved, _) = solve(&physical, &every_10s(2), &w, &[], &[]); + let selected = candidates(&solved.unwrap()); + assert!( + selected.contains(&physical_of(&physical, readouts[1])), + "{selected:?}" + ); +} + /// The same readout group as `q` every 10 s and as `scalar(q) * 2` every /// 20 s, both at relative 0.01, with only a readout at relative 0.01: the /// operator root accepts it, the scalar expression's operand only exact From 2a76cb3f4b6ab99f4359638193f74e61067034b1 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:25:06 +0000 Subject: [PATCH 2/5] test(accuracy): constraint (K) bounds the estimates each query's output depends on Tests first for the failure-probability rows of (K): a per-value bound P and a whole-output bound W per execution, with the root checking the bound Pass 2's split names (every estimate read, or each covered value). - Stage 3: or, concatenation, many-to-one match, and/unless and count inputs, exact inputs, per-value, whole-result and record-share scopes, an estimate reached on two paths, a top-k readout, a query with several roots, context copies; the reduction regression gets the equal-delta case (a per-value query at 0.01 gives a 0.01 copy). - A brute-force oracle over small shapes and every dataset of two 6-series metrics: every accepted plan's union-bound worst case is within delta, with random looser copies; every shape alone has a plan whenever it has one without delta. - asap-types: AccuracyTarget::failure_probability_bound and Memo::max_estimates_per_evaluation (top-k counts every item it ranks). - Integration: or of two quantiles, group_left onto every series, max reading its own split's level. These use new interfaces and do not compile without the next commit. --- .../tests/accuracy_at_query_roots.rs | 149 +++- .../src/milp/model/tests.rs | 662 ++++++++++++++++-- .../milp/model/tests/union_bound_oracle.rs | 591 ++++++++++++++++ crates/types/src/accuracy/target/tests.rs | 158 ++++- crates/types/src/stages/logical/memo/tests.rs | 74 ++ 5 files changed, 1549 insertions(+), 85 deletions(-) create mode 100644 crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs diff --git a/crates/integration-tests/tests/accuracy_at_query_roots.rs b/crates/integration-tests/tests/accuracy_at_query_roots.rs index 3b8292de6..3f1b6bd75 100644 --- a/crates/integration-tests/tests/accuracy_at_query_roots.rs +++ b/crates/integration-tests/tests/accuracy_at_query_roots.rs @@ -4,7 +4,8 @@ //! error 0.01, and B, the ratio of its p99 and p50 at relative error 0.03. use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; -use asap_planner::{DeploymentCapabilities, PlanningModels}; +use asap_planner::{DeploymentCapabilities, PlanningModels, PlanningRun}; +use asap_types::ir::{ASAPOp, Operator}; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::summary_type::SketchParams; use asap_types::workload::PlanningWorkload; @@ -16,6 +17,16 @@ const LATENCY_SERIES: usize = 1_000; /// `latency` (`series_count` series, one sample per second each, /// ingested, values declared non-negative). fn workload(queries: &[(&str, &str)], series_count: usize) -> PlanningWorkload { + workload_with_sources(queries, series_count, "") +} + +/// [`workload`] with `more_sources`, JSON members of `sources` after +/// `latency`'s (each starting with a comma). +fn workload_with_sources( + queries: &[(&str, &str)], + series_count: usize, + more_sources: &str, +) -> PlanningWorkload { let entries: Vec = queries .iter() .map(|(query, accuracy)| { @@ -29,7 +40,7 @@ fn workload(queries: &[(&str, &str)], series_count: usize) -> PlanningWorkload { r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": {series_count}.0}}}}, "sources": {{"latency": {{"series_count": {series_count}, "metric_type": "gauge", "sample_interval": 1000, - "column_signs": {{"value": "non_negative"}}}}}}}}}}"#, + "column_signs": {{"value": "non_negative"}}}}{more_sources}}}}}}}"#, entries.join(",") )) .unwrap() @@ -52,7 +63,19 @@ fn selection( series_count: usize, ignore_accuracy: bool, ) -> (PhysicalASAPDAG, Vec) { - let workload = workload(queries, series_count); + let run = planning_run(workload(queries, series_count), ignore_accuracy); + let (plan, selection) = run.selection.as_ref().expect("a plan is selected"); + let accepted = selection + .accepted_accuracy_per_query + .iter() + .map(ToString::to_string) + .collect(); + (plan.clone(), accepted) +} + +/// Every planning stage over `workload`, no raw data kept, with accuracy +/// enforced or ignored. +fn planning_run(workload: PlanningWorkload, ignore_accuracy: bool) -> PlanningRun { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap(); let capabilities = DeploymentCapabilities { @@ -63,14 +86,31 @@ fn selection( .with_deployment_capabilities(&capabilities) .with_ignore_accuracy_guarantees(ignore_accuracy); let roots = lower_promql_roots(&workload).unwrap(); - let run = run_planning_stages_over(roots, &entries, &data, models).unwrap(); - let (plan, selection) = run.run.selection.as_ref().expect("a plan is selected"); - let accepted = selection - .accepted_accuracy_per_query + run_planning_stages_over(roots, &entries, &data, models) + .unwrap() + .run +} + +/// The failure probability each readout the selected plan reads was +/// required to meet (`CandidateForMemoGroup::required_accuracy`), in node +/// order. +fn readout_failure_probabilities(run: &PlanningRun) -> Vec { + let (plan, _) = run.selection.as_ref().expect("a plan is selected"); + let physical = run.physical_asap_candidates.as_ref().expect("Stage 2 ran"); + plan.nodes .iter() - .map(ToString::to_string) - .collect(); - (plan.clone(), accepted) + .filter(|node| matches!(node.payload, Operator::ASAP(ASAPOp::SummaryEstimate { .. }))) + .filter_map(|node| { + let candidate = physical.physical_candidates[node.selected_physical_candidate?.0] + .logical_candidate_id; + let required = physical + .logical_candidates + .memo + .candidate(candidate) + .required_accuracy()?; + Some(required.failure_probability) + }) + .collect() } /// Queries A and B of the module's example. @@ -221,3 +261,92 @@ fn a_ratio_sharing_its_readouts_with_its_operands_still_meets_its_target() { "the ratio reads readouts composing within 0.05: {alphas:?}" ); } + +/// The p90 of `latency{side="a"}` or of `latency{side="b"}`, over the +/// whole result at relative 0.05 with δ = 0.01. Stage 0 bounds the result +/// by the series both branches read, 2,000, and the readouts read 2,000 +/// estimates, one per output value: every estimate read bounds it, so +/// Pass 2 gives each readout 0.01 / 2,000 = 5e-6 and (K) accepts them +/// (2,000 × 5e-6 = 0.01). Adding both sides' δ per output value against +/// 0.01 / 2,000 left this query no sketch plan. +#[test] +fn an_or_of_two_quantiles_reads_sketches() { + let run = planning_run( + workload( + &[( + r#"quantile_over_time(0.9, latency{side="a"}[5m]) or quantile_over_time(0.9, latency{side="b"}[5m])"#, + r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01}"#, + )], + LATENCY_SERIES, + ), + false, + ); + let (_, selection) = run.selection.as_ref().expect("a plan is selected"); + assert_eq!( + selection.accepted_accuracy_per_query[0].to_string(), + "accepted relative (DDSketch)" + ); + let levels = readout_failure_probabilities(&run); + assert!(!levels.is_empty()); + for level in levels { + assert!((level - 5e-6).abs() <= 1e-12, "{level}"); + } +} + +/// `max_over_time(latency[5m]) * on() group_left +/// quantile_over_time(0.99, reference[5m])`, `reference` one series +/// matched onto each of `latency`'s 1,000 (read exactly), over the +/// whole result at relative 0.01 with δ = 0.01: every output value reads +/// the one estimate, so the readout keeps δ = 0.01 and (K) accepts it. +/// Splitting δ over the 1,000 output values left it no sketch plan. +#[test] +fn a_quantile_matched_onto_every_series_reads_a_sketch() { + let run = planning_run( + workload_with_sources( + &[( + "max_over_time(latency[5m]) * on() group_left quantile_over_time(0.99, reference[5m])", + r#"{"accuracy_metric": "relative", "max_error": 0.02, "failure_probability": 0.01}"#, + )], + LATENCY_SERIES, + r#", "reference": {"series_count": 1, "metric_type": "gauge", "sample_interval": 1000, + "column_signs": {"value": "non_negative"}}"#, + ), + false, + ); + let (_, selection) = run.selection.as_ref().expect("a plan is selected"); + assert_eq!( + selection.accepted_accuracy_per_query[0].to_string(), + "accepted relative (DDSketch)" + ); + assert_eq!(readout_failure_probabilities(&run), [0.01]); +} + +/// The p90 of `latency` per value at δ = 0.01 (its own level 0.01) and +/// `max` of it over the whole result at δ = 0.01 (its own level 1e-5): +/// `max` reads all 1,000 estimates, so the readout it reads must fail +/// with at most 1e-5, whatever level the per-value query was given. +#[test] +fn a_max_reads_the_level_its_own_split_gives() { + let run = planning_run( + workload( + &[ + ( + "quantile_over_time(0.9, latency[5m])", + r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01, "accuracy_scope": "per_value"}"#, + ), + ( + "max(quantile_over_time(0.9, latency[5m]))", + r#"{"accuracy_metric": "relative", "max_error": 0.05, "failure_probability": 0.01}"#, + ), + ], + LATENCY_SERIES, + ), + false, + ); + let levels = readout_failure_probabilities(&run); + assert!(!levels.is_empty()); + assert!( + levels.iter().all(|&level| level <= 1e-5 * (1.0 + 1e-9)), + "{levels:?}" + ); +} diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 038e0d5bb..365d98210 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -4,15 +4,21 @@ use std::collections::{BTreeSet, HashMap}; -use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; +use asap_types::accuracy::{ + AccuracyMetric, AccuracyScope, AccuracyTarget, FailureProbabilityBound, +}; +use asap_types::ir::operator::arguments::{ + BinaryOpKind, GroupSide, PromQLVectorSetOpKind, VectorGrouping, VectorMatch, VectorMatchKind, +}; use asap_types::ir::operator::{AggIntent, Reduction}; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::expr_ir::CompareOpKind; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::scalar::ExprSemantics; use asap_types::ir::scalar::ScalarValue; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, Operator}; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; use asap_types::ir::{QueryRoot, ScalarExpr}; use asap_types::stages::{ CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalRequirements, @@ -29,7 +35,7 @@ use asap_types::test_support::memo_builders::{ }; use asap_types::test_support::memo_fixtures; use asap_types::test_support::memo_fixtures::arithmetic; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, SourceFacts}; use asap_types::workload::{ Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, TimestampMs, @@ -1225,15 +1231,31 @@ fn ddsketch_readouts( q: f64, levels: &[(f64, f64)], ) -> (MemoGroupId, Vec) { - let m = scan(memo); - let window = range(memo, 60, m); - let state = sketch_state( + readouts_of( memo, - window, + "m", SketchAlgorithm::DDSketch, - Reduction::PerEntity, - ); - let operator = memo_fixtures::readout(SketchStatistic::Quantile { q }, state); + SketchStatistic::Quantile { q }, + AccuracyMetric::Relative, + levels, + ) +} + +/// `statistic` over a state of `algorithm` over `metric[1m]` per series: +/// the readout group, one readout per requirement in `error_metric` of +/// `levels` (ε, δ). +fn readouts_of( + memo: &mut Memo, + metric: &str, + algorithm: SketchAlgorithm, + statistic: SketchStatistic, + error_metric: AccuracyMetric, + levels: &[(f64, f64)], +) -> (MemoGroupId, Vec) { + let rows = scan_group(memo, metric, &["job"], vec![]); + let window = range(memo, 60, rows); + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let operator = memo_fixtures::readout(statistic, state); let group = memo.find_or_add_group(operator.clone()).unwrap(); let readouts = levels .iter() @@ -1246,7 +1268,7 @@ fn ddsketch_readouts( ); readout.input_requirements = vec![LogicalRequirements { required_accuracy: Some(RequiredAccuracy { - metric: AccuracyMetric::Relative, + metric: error_metric, max_error, failure_probability, }), @@ -1257,6 +1279,25 @@ fn ddsketch_readouts( (group, readouts) } +/// A memo over sources each declaring only its series count. +fn memo_with_series(series: &[(&str, u64)]) -> Memo { + Memo::new(&DataWorkload { + sources: series + .iter() + .map(|&(metric, series_count)| { + ( + metric.to_string(), + SourceFacts { + series_count: Some(series_count), + ..SourceFacts::default() + }, + ) + }) + .collect(), + ..DataWorkload::default() + }) +} + /// Every candidate of `memo` at query time, `OneEvaluation`. fn every_candidate(memo: &Memo) -> Vec<(CandidateForMemoGroupId, Lifetime)> { memo.groups() @@ -1297,11 +1338,21 @@ fn relative(max_error: f64) -> AccuracyTarget { /// and misses the target, so it bounds nothing: as a bound it would leave /// out each 0.01 readout (cost 3 > 2). The same ratio every 10 s and every /// 20 s is one context's copy, whose (K) has no rows of its own and -/// selects alike. +/// selects alike. The ratio's only variable is its error $E$: the target +/// sets no failure probability, so no root checks one, with or without +/// `m`'s series declared. #[test] fn k_allocates_a_ratio_s_error_among_its_readouts() { - for entries in [vec![every(10_000)], vec![every(10_000), every(20_000)]] { - let mut memo = empty_memo(); + for (entries, series_declared) in [ + (vec![every(10_000)], false), + (vec![every(10_000), every(20_000)], false), + (vec![every(10_000)], true), + ] { + let mut memo = if series_declared { + memo_with_series(&[("m", 1)]) + } else { + empty_memo() + }; let levels = [(0.02, 1.0), (0.01, 1.0)]; let (p99, p99_readouts) = ddsketch_readouts(&mut memo, 0.99, &levels); let (p50, p50_readouts) = ddsketch_readouts(&mut memo, 0.5, &levels); @@ -1344,7 +1395,7 @@ fn k_allocates_a_ratio_s_error_among_its_readouts() { ); assert!(!selected.contains(¬_chosen), "{selected:?}"); } - // (K)'s E and D exist only in the 10 s context (the 20 s one is + // (K)'s variables exist only in the 10 s context (the 20 s one is // its copy), and only for the ratio: a readout's error is fixed. let kept = vec![true; selectable.executions().len()]; let rules = AccuracyRules::of(&inputs).unwrap(); @@ -1359,7 +1410,7 @@ fn k_allocates_a_ratio_s_error_among_its_readouts() { ); let model = ErrorModel::new(&inputs, &rules, &variables.y, &shared); let accuracy = AccuracyVariables::new(&model, &mut problem); - let one_context = 2; + let one_context = 1; assert_eq!(accuracy.variable_count(), one_context, "{entries:?}"); } } @@ -1404,21 +1455,6 @@ fn k_accepts_error_only_in_the_target_s_metric() { assert!(plans(Some(any), &level, Some(1), operator)); } -/// A whole-result failure probability 0.01 over 10 values is 1e-3 per -/// value, which a readout failing with probability 1e-3 meets; over 100 -/// values (1e-4 per value) it does not. With the count unknown no -/// per-value δ follows, and (K) has no row for it (Pass 2 gives such a -/// query's readouts no level). -#[test] -fn k_divides_a_whole_result_failure_probability_over_the_values() { - let level = [(0.01, 1e-3)]; - let target = relative(0.01).failure_probability(0.01); - let operator = QueryRoot::Operator; - assert!(plans(Some(target.clone()), &level, Some(10), operator)); - assert!(!plans(Some(target.clone()), &level, Some(100), operator)); - assert!(plans(Some(target), &level, None, operator)); -} - /// `scalar(v)` returns the value it reads, so its root accepts the /// target's metric; `scalar(v) * 2`'s operand is read exactly, as no error /// contract covers the scalar expression; without a target either reads @@ -1468,8 +1504,9 @@ fn k_caps_a_ratio_s_denominator_at_its_validity_cap() { /// 0.01 over the whole result, returning 10 values every 10 s and 100 /// every 20 s: the readout failing with 1e-3 (cost 1) meets 1e-3 per /// value, not 1e-4, where the one failing with 1e-5 (cost 2) is needed. -/// The contexts differ in what (K) reads ($n_q$), so the 20 s one is not a -/// copy: each selects its own readout. +/// The series are undeclared, so each checks each covered value, 10 at +/// 1e-3 and 100 at 1e-4: the contexts differ in their bounds, so the 20 s +/// one is not a copy and each selects its own readout. #[test] fn contexts_whose_queries_split_their_failure_probability_differently_are_not_copies() { let mut memo = empty_memo(); @@ -1524,46 +1561,533 @@ fn contexts_whose_queries_split_their_failure_probability_differently_are_not_co /// estimates it reads does, so the 5e-4 copy gives it a failure /// probability up to 1,000 × 5e-4 = 0.5, not 0.01: only the 1e-5 copy /// (1,000 × 1e-5 = 0.01) meets its target. A rule adding the selected -/// readout's δ once accepts the 5e-4 copy for both queries. +/// readout's δ once accepts the 5e-4 copy for both queries. The same with +/// `q` per value at δ = 0.01, whose own level is 0.01 when δ splits over +/// the estimates one covered value depends on (1,000 × 0.01 = 10 for +/// `max(q)`). Either way `q` is cheapest reading the 1e-5 copy too, so +/// only that copy is selected. #[test] fn k_does_not_let_a_reduction_read_a_level_given_for_a_per_value_query() { - let mut memo = Memo::new(&with_series("m", 1_000)); - let (readout_group, readouts) = - ddsketch_readouts(&mut memo, 0.9, &[(0.01, 5e-4), (0.01, 1e-5)]); - let max = group( + for (per_value_delta, per_value_level) in [(0.5, 5e-4), (0.01, 0.01)] { + let mut memo = Memo::new(&with_series("m", 1_000)); + let (readout_group, readouts) = + ddsketch_readouts(&mut memo, 0.9, &[(0.01, per_value_level), (0.01, 1e-5)]); + let max = group( + &mut memo, + memo_fixtures::aggregate( + Reduction::by(vec![]), + AggIntent::Max { col: None }, + readout_group, + ), + ); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout_group), + accuracy_target: Some( + relative(0.01) + .failure_probability(per_value_delta) + .accuracy_scope(AccuracyScope::PerValue), + ), + number_of_values_per_query_evaluation: Some(1_000), + }); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(max), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + let physical = physical(memo.clone(), &every_candidate(&memo)); + // The scan costs 10, so neither readout is dearer than a feasible + // plan. + let mut w = vec![0.0; physical.physical_candidates.len()]; + w[0] = 10.0; + w[physical_of(&physical, readouts[0])] = 1.0; + w[physical_of(&physical, readouts[1])] = 3.0; + let (solved, _) = solve(&physical, &every_10s(2), &w, &[], &[]); + let selected = candidates(&solved.unwrap()); + assert!( + selected.contains(&physical_of(&physical, readouts[1])), + "{per_value_delta}: {selected:?}" + ); + } +} + +/// Whether a plan meets (K) for `memo`'s queries, every one every 10 s, +/// with every candidate at cost 1 but the scan of `m` (candidate 0) at +/// 10: the selected candidates, or `None` when (K) is infeasible. +fn selected_or_infeasible(memo: &Memo) -> Option> { + let physical = physical(memo.clone(), &every_candidate(memo)); + let mut w = vec![1.0; physical.physical_candidates.len()]; + w[0] = 10.0; + match solve(&physical, &every_10s(memo.queries().len()), &w, &[], &[]).0 { + Ok(solved) => Some(candidates(&solved)), + Err(SelectionMilpError::Infeasible) => None, + Err(other) => panic!("{other:?}"), + } +} + +/// A query rooted at `root` with `target` and $n_q$ `values`. +fn add_query(memo: &mut Memo, root: MemoGroupId, target: AccuracyTarget, values: Option) { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(root), + accuracy_target: Some(target), + number_of_values_per_query_evaluation: values, + }); +} + +/// `q or r`, p90 of `m` and of `r`, 100 series each with the same labels, +/// over a whole result of $n_q$ = 100 at δ = 0.01: each output value is +/// copied from one readout value, so the statement fails only when one of +/// at most 100 estimates does. With both readouts at 1e-4 (Pass 2's split, +/// 0.01 / min(200, 100 · 1)), each covered value fails with at most +/// max(1e-4, 1e-4), and 100 of them with 0.01: (K) accepts it. At 2e-4 +/// each, 0.02: it does not. Adding both sides' δ per value (2e-4 against +/// 1e-4) would leave `or` no sketch plan. +#[test] +fn k_accepts_an_or_whose_readouts_meet_delta_over_its_covered_values() { + for (level, plans) in [(1e-4, true), (2e-4, false)] { + let mut memo = memo_with_series(&[("m", 100), ("r", 100)]); + let p90 = SketchStatistic::Quantile { q: 0.9 }; + let relative_metric = AccuracyMetric::Relative; + let (of_m, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::DDSketch, + p90.clone(), + relative_metric, + &[(0.05, level)], + ); + let (of_r, _) = readouts_of( + &mut memo, + "r", + SketchAlgorithm::DDSketch, + p90, + relative_metric, + &[(0.05, level)], + ); + let or = group( + &mut memo, + memo_fixtures::set_operator(PromQLVectorSetOpKind::Or, of_m, of_r), + ); + add_query( + &mut memo, + or, + relative(0.05).failure_probability(0.01), + Some(100), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `q * on() group_left e`: `q` exact over 1000 series of `m`, `e` the p90 +/// of one series of `e`, over the whole result of 1000 values at δ = +/// 0.01. Every output value reads `e`'s one estimate, so the result fails +/// only when that estimate does: a readout failing with 0.01 meets it, +/// one failing with 0.02 does not. Splitting δ over the 1000 output +/// values (1e-5 each) would leave it no sketch plan. +#[test] +fn k_charges_a_readout_every_series_matches_once() { + for (level, plans) in [(0.01, true), (0.02, false)] { + let mut memo = memo_with_series(&[("m", 1_000), ("e", 1)]); + let rows = scan(&mut memo); + let window = range(&mut memo, 60, rows); + let q = group( + &mut memo, + memo_fixtures::aggregate(Reduction::PerEntity, AggIntent::Max { col: None }, window), + ); + let (e, _) = readouts_of( + &mut memo, + "e", + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.9 }, + AccuracyMetric::Relative, + &[(0.01, level)], + ); + let matched = group( + &mut memo, + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec![], + grouping: Some(VectorGrouping { + side: GroupSide::Left, + labels: vec![], + }), + }), + }, + return_bool: false, + lhs: q, + rhs: e, + }), + ); + add_query( + &mut memo, + matched, + relative(0.05).failure_probability(0.01), + Some(1_000), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `q and r` reads only which series `r` has, and `q / count(r)` none of +/// `r`'s values (β = 0 on `r`), so `r`'s readout, failing with 0.5 at a +/// level another query gave it, cannot make the output miss: over one +/// series at δ = 0.01, `q`'s readout at 0.01 meets the target alone. +#[test] +fn k_adds_nothing_for_an_input_whose_values_are_not_read() { + for division in [false, true] { + let mut memo = memo_with_series(&[("m", 1)]); + let (q, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.01, 0.01)]); + let (r, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 0.5)]); + let root = if division { + let count = group( + &mut memo, + memo_fixtures::aggregate(Reduction::by(vec![]), AggIntent::Count, r), + ); + group(&mut memo, arithmetic(ArithmeticOpKind::Div, q, count)) + } else { + group( + &mut memo, + memo_fixtures::set_operator(PromQLVectorSetOpKind::And, q, r), + ) + }; + add_query( + &mut memo, + root, + relative(0.05).failure_probability(0.01), + Some(1), + ); + assert!(selected_or_infeasible(&memo).is_some(), "{division}"); + } +} + +/// `q + r` with `q` exact and `r` a p90 readout at 0.01 (absolute error, +/// which a sum passes on), one series each: +/// an exact input has no estimate to fail, so the sum fails only with +/// `r`, 0.01, within δ = 0.01. The sum is not constant (one of its inputs +/// is a choice), so it has an error bound $E$ and the one failure +/// probability its root checks: `R` = $V$ · fan-in = 1 is a tie, so every +/// estimate read, $W$. Two variables. +#[test] +fn k_counts_an_exact_input_as_never_failing() { + let mut memo = memo_with_series(&[("m", 1)]); + let rows = scan(&mut memo); + let window = range(&mut memo, 60, rows); + let q = group( &mut memo, - memo_fixtures::aggregate( - Reduction::by(vec![]), - AggIntent::Max { col: None }, - readout_group, - ), + memo_fixtures::aggregate(Reduction::PerEntity, AggIntent::Max { col: None }, window), ); - memo.add_query(MemoQuery { - root: QueryRoot::Operator(readout_group), - accuracy_target: Some( + let (r, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.9 }, + AccuracyMetric::ABSOLUTE, + &[(0.01, 0.01)], + ); + let sum = group(&mut memo, arithmetic(ArithmeticOpKind::Add, q, r)); + add_query( + &mut memo, + sum, + AccuracyTarget::max_error(0.05) + .accuracy_metric(AccuracyMetric::ABSOLUTE) + .failure_probability(0.01), + Some(1), + ); + assert!(selected_or_infeasible(&memo).is_some()); + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let entries = every_10s(1); + let (selectable, costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs).unwrap(); + let shared = copies_in_equivalent_contexts(&inputs, &kept, Some(&rules)); + let mut problem = ProblemVariables::new(); + let variables = selection_variables( + &inputs, + &kept, + &shared, + &Components::of(&inputs), + &mut problem, + ); + let model = ErrorModel::new(&inputs, &rules, &variables.y, &shared); + assert_eq!( + AccuracyVariables::new(&model, &mut problem).variable_count(), + 2 + ); +} + +/// A readout over 100 series read per value at δ = 0.01: each value fails +/// only with its own estimate, so the readout at 0.01 meets it, whatever +/// the 100 values; one at 0.02 does not. +#[test] +fn k_does_not_divide_a_per_value_failure_probability_by_the_values() { + for (level, plans) in [(0.01, true), (0.02, false)] { + let mut memo = memo_with_series(&[("m", 100)]); + let (readout, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, level)]); + add_query( + &mut memo, + readout, relative(0.01) - .failure_probability(0.5) + .failure_probability(0.01) .accuracy_scope(AccuracyScope::PerValue), - ), - number_of_values_per_query_evaluation: Some(1_000), - }); - memo.add_query(MemoQuery { - root: QueryRoot::Operator(max), - accuracy_target: Some(relative(0.01).failure_probability(0.01)), - number_of_values_per_query_evaluation: Some(1), - }); + Some(100), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// A readout failing with 1e-3 read over the whole result at δ = 0.01. +/// With its series undeclared, (K) checks each covered value at δ / $n_q$: +/// 1e-3 over 10 values (10 · 1e-3), not over 100. With $n_q$ unknown too +/// no bound is known: (K) has no row, and the query reads the readout +/// group only exactly, by (X), so with only readouts in it no plan +/// exists. With its 10 series declared and $n_q$ unknown, every estimate +/// read bounds it: 10 · 1e-3 = 0.01. +#[test] +fn k_bounds_a_whole_result_by_its_covered_values_or_every_estimate() { + let target = relative(0.01).failure_probability(0.01); + let plans_over = |memo: Memo, values: Option, exact_only: bool| { + let mut memo = memo; + let (readout, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1e-3)]); + add_query(&mut memo, readout, target.clone(), values); + let mut physical = physical(memo.clone(), &every_candidate(&memo)); + if exact_only { + // (X) lists the readout group and every group below it. + let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = + memo.groups()[readout.0 as usize].expression + else { + unreachable!("a readout group") + }; + physical.logical_candidates.exact_only_groups_of_query = + vec![[readout, summary_input].into()]; + } + let w = vec![1.0; physical.physical_candidates.len()]; + match solve(&physical, &every_10s(1), &w, &[], &[]).0 { + Ok(_) => true, + Err(SelectionMilpError::Infeasible) => false, + Err(other) => panic!("{other:?}"), + } + }; + assert!(plans_over(empty_memo(), Some(10), false)); + assert!(!plans_over(empty_memo(), Some(100), false)); + assert!(!plans_over(empty_memo(), None, true)); + assert!(plans_over(memo_with_series(&[("m", 10)]), None, false)); +} + +/// A readout over 1000 series read over the groups holding at least 5% of +/// the records, at δ = 0.01: at most 20 groups qualify, so (K) checks +/// each of them at 0.01 / 20 = 5e-4. A readout at 5e-4 meets it; one at +/// 1e-3 does not. +#[test] +fn k_bounds_a_record_share_scope_by_its_covered_groups() { + for (level, plans) in [(5e-4, true), (1e-3, false)] { + let mut memo = memo_with_series(&[("m", 1_000)]); + let (readout, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, level)]); + add_query( + &mut memo, + readout, + relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::GroupsWithRecordShareAtLeast(0.05)), + Some(1_000), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `readout or {job="a"}(readout)`, the same p90 readout of 100 series on +/// both sides, over the whole result of 100 values at δ = 0.01: each +/// output value copies one estimate, so the readout at 1e-4 meets each +/// covered value (max(1e-4, 1e-4), 100 values: 0.01), although its two +/// paths read 200 estimates' worth. +#[test] +fn k_charges_an_estimate_both_sides_of_an_or_read_once_per_value() { + let mut memo = memo_with_series(&[("m", 100)]); + let (readout, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.05, 1e-4)]); + let or = or_of_readout_and_its_filter(&mut memo, readout); + add_query( + &mut memo, + or, + relative(0.05).failure_probability(0.01), + Some(100), + ); + assert!(selected_or_infeasible(&memo).is_some()); +} + +/// `readout or {job="a"}(readout)`. +fn or_of_readout_and_its_filter(memo: &mut Memo, readout: MemoGroupId) -> MemoGroupId { + let job_a = memo_fixtures::compare( + 2, + CompareOpKind::Eq, + ScalarValue::Utf8("a".into()), + ExprSemantics::Promql, + ); + let filtered = group(memo, memo_fixtures::filter(job_a, readout)); + group( + memo, + memo_fixtures::set_operator(PromQLVectorSetOpKind::Or, readout, filtered), + ) +} + +/// `max(readout or {job="a"}(readout))` over 100 series at δ = 0.01: the +/// max reads every estimate of both sides, 200 by path, so (K) checks +/// every estimate read and charges each estimate once per path: the +/// readout at 1e-4 gives 200 · 1e-4 = 0.02, and the dearer one at 5e-5 +/// (0.01) is selected. Only 100 distinct estimates can fail, so 1e-4 +/// would have been enough (`accuracy-models.md`, "What the rows lose", +/// item 1): a stricter δ than needed, never a missed target. +#[test] +fn k_charges_an_estimate_two_paths_reach_once_per_path() { + let mut memo = memo_with_series(&[("m", 100)]); + let (readout, readouts) = ddsketch_readouts(&mut memo, 0.9, &[(0.05, 1e-4), (0.05, 5e-5)]); + let or = or_of_readout_and_its_filter(&mut memo, readout); + let max = group( + &mut memo, + memo_fixtures::aggregate(Reduction::by(vec![]), AggIntent::Max { col: None }, or), + ); + add_query( + &mut memo, + max, + relative(0.05).failure_probability(0.01), + Some(1), + ); let physical = physical(memo.clone(), &every_candidate(&memo)); - // The scan costs 10, so neither readout is dearer than a feasible plan. let mut w = vec![0.0; physical.physical_candidates.len()]; w[0] = 10.0; w[physical_of(&physical, readouts[0])] = 1.0; - w[physical_of(&physical, readouts[1])] = 3.0; - let (solved, _) = solve(&physical, &every_10s(2), &w, &[], &[]); - let selected = candidates(&solved.unwrap()); + w[physical_of(&physical, readouts[1])] = 2.0; + let selected = candidates(&solve(&physical, &every_10s(1), &w, &[], &[]).0.unwrap()); assert!( selected.contains(&physical_of(&physical, readouts[1])), "{selected:?}" ); + assert!( + !selected.contains(&physical_of(&physical, readouts[0])), + "{selected:?}" + ); +} + +/// A top-k readout over 10 series, read per value at δ = 0.01: its output +/// keys are chosen by all 10 estimates it ranks, so one output value +/// fails when any of them does. The readout at 1e-3 meets it (10 · 1e-3), +/// at 2e-3 it does not. +#[test] +fn k_charges_a_top_k_readout_for_every_estimate_it_ranks() { + for (level, plans) in [(1e-3, true), (2e-3, false)] { + let mut memo = memo_with_series(&[("m", 10)]); + let (top_k, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 3 }, + AccuracyMetric::Relative, + &[(0.01, level)], + ); + add_query( + &mut memo, + top_k, + AccuracyTarget::max_error(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(30), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// The p90 and the p50 readouts of 100 series concatenated, per value at +/// δ = 0.01: each row comes from one readout, so it fails only with that +/// readout's estimate: readouts at 0.01 meet it, at 0.02 they do not. +#[test] +fn k_charges_a_concatenation_one_input_per_value() { + for (level, plans) in [(0.01, true), (0.02, false)] { + let mut memo = memo_with_series(&[("m", 100)]); + let (p90, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.01, level)]); + let (p50, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, level)]); + let both = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Concat { + children: vec![p90, p50], + }), + ); + add_query( + &mut memo, + both, + relative(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(200), + ); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// `scalar(p90) + scalar(p50)` over one series at δ = 0.01, each readout +/// required with no error (the operands of a scalar expression no +/// contract covers are read exactly) but failing with `level`: the sum +/// fails when either does, so its one row adds both roots' failure +/// probabilities. 0.005 each meets 0.01; 0.006 each (0.012) does not, +/// although each root alone is within 0.01. +#[test] +fn k_sums_the_failure_probabilities_of_a_query_s_roots() { + for (level, plans) in [(0.005, true), (0.006, false)] { + let mut memo = memo_with_series(&[("m", 1)]); + let (p90, _) = ddsketch_readouts(&mut memo, 0.9, &[(0.0, level)]); + let (p50, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.0, level)]); + memo.add_query(MemoQuery { + root: QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Add, + left: Box::new(ScalarExpr::PromqlScalarFromVector(p90)), + right: Box::new(ScalarExpr::PromqlScalarFromVector(p50)), + semantics: ExprSemantics::Promql, + }), + accuracy_target: Some(relative(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(1), + }); + assert_eq!(selected_or_infeasible(&memo).is_some(), plans, "{level}"); + } +} + +/// The same readout over 10 series every 10 s and every 20 s, declaring +/// 10 and 50 values, both over the whole result at δ = 0.01: each reads +/// its 10 estimates (no more than its values), so both check every +/// estimate read against 0.01, and the 20 s context is a copy. +#[test] +fn contexts_whose_queries_bound_every_estimate_read_at_equal_delta_are_copies() { + let mut memo = memo_with_series(&[("m", 10)]); + let (readout_group, _) = ddsketch_readouts(&mut memo, 0.5, &[(0.01, 1e-3)]); + for values in [10, 50] { + add_query( + &mut memo, + readout_group, + relative(0.01).failure_probability(0.01), + Some(values), + ); + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + let w = vec![1.0; physical.physical_candidates.len()]; + let entries = [every(10_000), every(20_000)]; + let (selectable, costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let kept = vec![true; selectable.executions().len()]; + let rules = AccuracyRules::of(&inputs); + let shared = copies_in_equivalent_contexts(&inputs, &kept, rules.as_ref()); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(0)]); } /// The same readout group as `q` every 10 s and as `scalar(q) * 2` every @@ -1705,20 +2229,26 @@ fn a_selection_missing_its_target_at_the_default_tolerance_is_solved_again_at_th /// first solve selects two (1.08e-6, within HiGHS's default tolerance of /// the 1e-6 limit), the exact check finds the miss, and at tolerance 1e-10 /// no plan meets the limit, so HiGHS reports the MILP infeasible. The -/// error names the query the first solve missed and its per-value δ, -/// not a plan-less "infeasible" that never mentions accuracy. +/// error names the query the first solve missed and its bound: each of +/// the 1e4 covered values at 1e-6 (the readouts' values per evaluation +/// are unknown, so every estimate read bounds nothing), not a plan-less +/// "infeasible" that never mentions accuracy. #[test] fn a_target_no_plan_meets_once_solved_at_the_least_tolerance_names_the_missed_query() { let solved = solve_ratio_over_values(10_000, [5.4e-7, 5.4e-7]); let Err(SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_per_output_value: Some(delta), + failure_probability_bound: + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 10_000, + limit, + }), }) = solved else { panic!("{solved:?}"); }; assert_eq!(query, QueryId(0)); - assert!((delta - 1e-6).abs() <= 1e-12, "{delta}"); + assert!((limit - 1e-6).abs() <= 1e-12, "{limit}"); } /// The same ratio over readouts failing with 4e-7 (cost 1) or 3e-7 @@ -1746,12 +2276,14 @@ fn a_per_value_failure_probability_below_the_solver_precision_is_an_error() { let solved = solve_ratio_over_values(values, failure_probabilities); let Err(SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_per_output_value: Some(delta), + failure_probability_bound: Some(FailureProbabilityBound::EachCoveredValue { limit, .. }), }) = solved else { panic!("{values}: {solved:?}"); }; assert_eq!(query, QueryId(0)); - assert!((delta - per_value).abs() <= 1e-6 * per_value, "{delta}"); + assert!((limit - per_value).abs() <= 1e-6 * per_value, "{limit}"); } } + +mod union_bound_oracle; diff --git a/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs b/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs new file mode 100644 index 000000000..fae29e0bc --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs @@ -0,0 +1,591 @@ +//! A brute-force check of constraint (K)'s failure-probability rows +//! (`accuracy-models.md`, "Which estimates a result depends on"): for +//! every plan the MILP accepts, the union-bound sum over the estimates the +//! covered output values depend on, maximized over every dataset the +//! declared facts allow, is at most the query's δ. +//! +//! It computes the union-bound worst case, `max_D Σ_{e ∈ S_q(D)} δ_e`, not +//! the true failure probability (which also depends on how the estimates' +//! failures correlate); that sum is what the rows must bound. +//! +//! Each shape is built over two metrics `m` and `b`, each declaring 6 +//! series (labels `job` ∈ {0, 1} and `instance` ∈ {0, 1, 2}); a dataset is +//! a subset of each metric's 6 series (64 × 64 datasets). Each query's +//! selected path is evaluated over every dataset, every output value +//! carrying the estimates (readout copy, series) it depends on: +//! - a readout's value of a series depends on that estimate; +//! - an exact per-series value on none; +//! - a filter keeps `job = 0`; +//! - `or` copies the left value, else the right one; `and` and `unless` +//! copy the left value where the right does (not) have the series; +//! - `+` one to one unions both sides' sets; `+ on() group_left` adds the +//! right side's only value (no output unless it has exactly one); +//! - `max` and `avg by (job)` union their group's sets. +//! +//! Each readout group gets the level Pass 2's split gives each query +//! reaching it (`AccuracyTarget::failure_probability_per_readout_value` +//! over `asap_accuracy::readout_value_counts_of_queries`) and random +//! further copies, some looser and cheaper, so the solver is tempted by +//! copies given for another query. Queries are planned alone and in pairs +//! (a per-value query on a readout with a query over a shape reading it), +//! under the whole-result, per-value and record-share scopes. + +use std::collections::{BTreeMap, BTreeSet}; + +use asap_accuracy::{readout_value_counts_of_queries, ErrorMetricsOfMemo}; +use asap_types::workload::bounds::source_name; + +use super::*; + +/// A series of `m` or `b`: (job, instance). +type Series = (u8, u8); + +/// An output value's key: a series, a job, or nothing (a global +/// reduction). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Key { + Series(Series), + Job(u8), + Nothing, +} + +/// An estimate: the readout execution's physical candidate and the +/// series. +type Estimate = (usize, Series); + +/// One evaluation's output: per key, the estimates the value depends on. +type Values = BTreeMap>; + +const SERIES: [Series; 6] = [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)]; + +/// A small deterministic generator (SplitMix64). +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// A PromQL set operator. +#[derive(Debug, Clone, Copy)] +enum SetKind { + Or, + And, + Unless, +} + +impl SetKind { + fn operator(self) -> PromQLVectorSetOpKind { + match self { + Self::Or => PromQLVectorSetOpKind::Or, + Self::And => PromQLVectorSetOpKind::And, + Self::Unless => PromQLVectorSetOpKind::Unless, + } + } +} + +/// The query shapes, as builders over a memo. +#[derive(Debug, Clone, Copy)] +enum Shape { + Readout(&'static str), + Exact(&'static str), + Filter(&'static Shape), + Max(&'static Shape), + AvgByJob(&'static Shape), + Set(SetKind, &'static Shape, &'static Shape), + Add(&'static Shape, &'static Shape), + /// `left + on() group_left max(right)`. + AddMaxOfRight(&'static Shape, &'static Shape), +} + +const RM: Shape = Shape::Readout("m"); +const RB: Shape = Shape::Readout("b"); +const EM: Shape = Shape::Exact("m"); +const FILTER_RM: Shape = Shape::Filter(&RM); +const OR: Shape = Shape::Set(SetKind::Or, &RM, &RB); +const OR_SELF: Shape = Shape::Set(SetKind::Or, &RM, &FILTER_RM); +const AND: Shape = Shape::Set(SetKind::And, &RM, &RB); +const UNLESS: Shape = Shape::Set(SetKind::Unless, &RM, &RB); +const ADD: Shape = Shape::Add(&RM, &RB); +const ADD_EXACT: Shape = Shape::Add(&EM, &RB); +const GROUP_LEFT: Shape = Shape::AddMaxOfRight(&EM, &RB); +const GROUP_LEFT_READOUTS: Shape = Shape::AddMaxOfRight(&RM, &RB); + +/// Every shape of depth at most 3 the oracle covers. +const SHAPES: &[Shape] = &[ + RM, + FILTER_RM, + Shape::Max(&RM), + Shape::AvgByJob(&RM), + OR, + OR_SELF, + AND, + UNLESS, + ADD, + ADD_EXACT, + GROUP_LEFT, + GROUP_LEFT_READOUTS, + Shape::Max(&OR), + Shape::Max(&OR_SELF), + Shape::AvgByJob(&OR), + Shape::Max(&ADD), + Shape::AvgByJob(&ADD_EXACT), + Shape::Max(&AND), + Shape::Filter(&OR), + Shape::Set(SetKind::Or, &ADD, &RB), + Shape::Add(&OR, &RB), +]; + +/// The column of label `label` in `group`'s schema. +fn column_of(memo: &Memo, group: MemoGroupId, label: &str) -> usize { + memo.groups()[group.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == label) + .unwrap() +} + +/// Builds `shape` in `memo`; the readout groups it adds are listed in +/// `readouts` (each with one candidate without a requirement, as Pass 1 +/// leaves it). +fn build(memo: &mut Memo, shape: Shape, readouts: &mut BTreeSet) -> MemoGroupId { + let built = match shape { + Shape::Readout(metric) => { + let rows = scan_group(memo, metric, &["job", "instance"], vec![]); + let window = range(memo, 60, rows); + let state = sketch_state( + memo, + window, + SketchAlgorithm::DDSketch, + Reduction::PerEntity, + ); + let operator = memo_fixtures::readout(SketchStatistic::Quantile { q: 0.9 }, state); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + if readouts.insert(group) { + try_candidate(memo, group, operator, None).unwrap(); + } + return group; + } + Shape::Exact(metric) => { + let rows = scan_group(memo, metric, &["job", "instance"], vec![]); + let window = range(memo, 60, rows); + memo_fixtures::aggregate(Reduction::PerEntity, AggIntent::Max { col: None }, window) + } + Shape::Filter(child) => { + let child = build(memo, *child, readouts); + let job_0 = memo_fixtures::compare( + column_of(memo, child, "job"), + CompareOpKind::Eq, + ScalarValue::Utf8("0".into()), + ExprSemantics::Promql, + ); + memo_fixtures::filter(job_0, child) + } + Shape::Max(child) => { + let child = build(memo, *child, readouts); + memo_fixtures::aggregate(Reduction::by(vec![]), AggIntent::Max { col: None }, child) + } + Shape::AvgByJob(child) => { + let child = build(memo, *child, readouts); + let job = column_of(memo, child, "job"); + memo_fixtures::aggregate( + Reduction::by(vec![job]), + AggIntent::Avg { col: None }, + child, + ) + } + Shape::Set(kind, left, right) => { + let (left, right) = (build(memo, *left, readouts), build(memo, *right, readouts)); + memo_fixtures::set_operator(kind.operator(), left, right) + } + Shape::Add(left, right) => { + let (left, right) = (build(memo, *left, readouts), build(memo, *right, readouts)); + arithmetic(ArithmeticOpKind::Add, left, right) + } + Shape::AddMaxOfRight(left, right) => { + let left = build(memo, *left, readouts); + let right = build(memo, Shape::Max(right), readouts); + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec![], + grouping: Some(VectorGrouping { + side: GroupSide::Left, + labels: vec![], + }), + }), + }, + return_bool: false, + lhs: left, + rhs: right, + }) + } + }; + group(memo, built) +} + +/// What one evaluation of execution `v` returns over `dataset` (the +/// series of each metric present), with `producer_of` the selected +/// producer per (consumer, input). +fn evaluate( + physical: &PhysicalASAPMemoCandidates, + producer_of: &BTreeMap<(usize, usize), usize>, + v: usize, + dataset: &BTreeMap<&str, Vec>, +) -> Values { + let memo = &physical.logical_candidates.memo; + let id = physical.physical_candidates[v].logical_candidate_id; + let candidate = memo.candidate(id); + let input = |i: usize| evaluate(physical, producer_of, producer_of[&(v, i)], dataset); + let series_of_group = || { + let source = memo.groups()[id.group.0 as usize] + .group_properties + .scanned_sources + .iter() + .next() + .unwrap(); + dataset[source_name(source)].clone() + }; + match &candidate.expression.operator { + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) => series_of_group() + .into_iter() + .map(|series| (Key::Series(series), [(v, series)].into())) + .collect(), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + .. + }) => series_of_group() + .into_iter() + .map(|series| (Key::Series(series), BTreeSet::new())) + .collect(), + Operator::NonASAP(NonASAPOp::Filter { .. }) => input(0) + .into_iter() + .filter(|(key, _)| matches!(key, Key::Series((0, _)))) + .collect(), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(by), + .. + }) => { + let mut out = Values::new(); + for (key, estimates) in input(0) { + let group = match (by.keys().is_empty(), key) { + (true, _) => Key::Nothing, + (false, Key::Series((job, _)) | Key::Job(job)) => Key::Job(job), + (false, Key::Nothing) => Key::Nothing, + }; + out.entry(group).or_default().extend(estimates); + } + out + } + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + kind, vector_match, .. + }, + .. + }) => { + let (left, right) = (input(0), input(1)); + match (kind, vector_match) { + (BinaryOpKind::Set(PromQLVectorSetOpKind::Or), _) => { + let mut out = right; + out.extend(left); + out + } + (BinaryOpKind::Set(PromQLVectorSetOpKind::And), _) => left + .into_iter() + .filter(|(key, _)| right.contains_key(key)) + .collect(), + (BinaryOpKind::Set(PromQLVectorSetOpKind::Unless), _) => left + .into_iter() + .filter(|(key, _)| !right.contains_key(key)) + .collect(), + (_, Some(_)) => { + // `on() group_left`: the right side's only value. + let mut only = right.into_values(); + match (only.next(), only.next()) { + (Some(matched), None) => left + .into_iter() + .map(|(key, mut estimates)| { + estimates.extend(matched.iter().copied()); + (key, estimates) + }) + .collect(), + _ => Values::new(), + } + } + (_, None) => left + .into_iter() + .filter_map(|(key, mut estimates)| { + estimates.extend(right.get(&key)?.iter().copied()); + Some((key, estimates)) + }) + .collect(), + } + } + other => panic!("the oracle does not evaluate {other:?}"), + } +} + +/// The union-bound worst case of query `query`'s selected path at `v` +/// over every dataset: the largest summed δ of the estimates the values +/// its scope covers depend on. +fn worst_union_bound( + physical: &PhysicalASAPMemoCandidates, + producer_of: &BTreeMap<(usize, usize), usize>, + v: usize, + scope: AccuracyScope, +) -> f64 { + let memo = &physical.logical_candidates.memo; + let delta_of = |copy: usize| { + let id = physical.physical_candidates[copy].logical_candidate_id; + memo.candidate(id) + .required_accuracy() + .map_or(0.0, |required| required.failure_probability) + }; + let mut worst: f64 = 0.0; + for m_subset in 0u32..64 { + for b_subset in 0u32..64 { + let present = |subset: u32| -> Vec { + (0..6) + .filter(|bit| subset & (1 << bit) != 0) + .map(|bit| SERIES[bit]) + .collect() + }; + let dataset: BTreeMap<&str, Vec> = + [("m", present(m_subset)), ("b", present(b_subset))].into(); + let values: Vec> = evaluate(physical, producer_of, v, &dataset) + .into_values() + .collect(); + let sum = |covered: &[&BTreeSet]| -> f64 { + let union: BTreeSet = + covered.iter().flat_map(|set| set.iter().copied()).collect(); + union.iter().map(|&(copy, _)| delta_of(copy)).sum() + }; + let covered = match scope { + AccuracyScope::PerValue => values.iter().map(|set| sum(&[set])).fold(0.0, f64::max), + AccuracyScope::WholeResult => sum(&values.iter().collect::>()), + AccuracyScope::GroupsWithRecordShareAtLeast(share) => { + // Which groups qualify depends on exact record counts: + // any `⌊1 / share⌋` of them may. + let at_most = (1.0 / share).floor() as usize; + (0u32..1 << values.len()) + .filter(|chosen| chosen.count_ones() as usize <= at_most) + .map(|chosen| { + let subset: Vec<&BTreeSet> = values + .iter() + .enumerate() + .filter(|(index, _)| chosen & (1 << index) != 0) + .map(|(_, set)| set) + .collect(); + sum(&subset) + }) + .fold(0.0, f64::max) + } + }; + worst = worst.max(covered); + } + } + worst +} + +/// The most values one evaluation of `shape` returns over the datasets, +/// $n_q$ as Stage 0 would bound it (at least 1). +fn values_per_evaluation(shape: Shape) -> u64 { + match shape { + Shape::Readout(_) | Shape::Exact(_) | Shape::Filter(_) => 6, + Shape::Max(_) => 1, + Shape::AvgByJob(_) => 2, + Shape::Set(..) | Shape::Add(..) | Shape::AddMaxOfRight(..) => 6, + } +} + +/// One planning case: the queries (shape, scope), with `delta` each (no +/// failure probability when `None`, which only checks that a plan +/// exists). +fn check( + queries: &[(Shape, AccuracyScope)], + delta: Option, + rng: &mut Rng, + report: &mut OracleReport, +) { + let mut memo = memo_with_series(&[("m", 6), ("b", 6)]); + let mut readouts = BTreeSet::new(); + let mut roots = Vec::new(); + for &(shape, scope) in queries { + let root = build(&mut memo, shape, &mut readouts); + roots.push(root); + let mut target = AccuracyTarget::max_error(1.0) + .accuracy_metric(AccuracyMetric::ABSOLUTE) + .accuracy_scope(scope); + if let Some(delta) = delta { + target = target.failure_probability(delta); + } + add_query(&mut memo, root, target, Some(values_per_evaluation(shape))); + } + // Pass 2's level per query reaching each readout group, plus random + // copies at δ from 1e-5 to 0.5. + let order = memo.topological_order().unwrap(); + let counts = + readout_value_counts_of_queries(&memo, &ErrorMetricsOfMemo::of(&memo, &order), &order); + for &readout in &readouts { + let mut levels: Vec = memo.groups()[readout.0 as usize] + .reached_by_queries() + .iter() + .filter_map(|&query| { + let memo_query = &memo.queries()[query]; + memo_query + .accuracy_target + .as_ref()? + .failure_probability_per_readout_value( + memo_query.number_of_values_per_query_evaluation, + counts[&query], + ) + }) + .collect(); + for _ in 0..2 { + levels.push([1e-5, 1e-4, 1e-3, 5e-3, 0.01, 0.05, 0.5][rng.below(7) as usize]); + } + let pass1 = CandidateForMemoGroupId { + group: readout, + index_in_group: 0, + }; + let unset = memo.candidate(pass1).clone(); + let requirement = |failure_probability| { + vec![LogicalRequirements { + required_accuracy: Some(RequiredAccuracy { + metric: AccuracyMetric::ABSOLUTE, + max_error: 0.01, + failure_probability, + }), + }] + }; + memo.set_candidate_input_requirements(pass1, requirement(levels[0])); + for &level in &levels[1..] { + let mut copy = unset.clone(); + copy.input_requirements = requirement(level); + let _ = memo.add_candidate(readout, copy); + } + } + let physical = physical(memo.clone(), &every_candidate(&memo)); + // Random costs; a looser copy tends to be cheaper. + let w: Vec = physical + .physical_candidates + .iter() + .map(|candidate| { + let looseness = memo + .candidate(candidate.logical_candidate_id) + .required_accuracy() + .map_or(0.0, |required| required.failure_probability.log10() + 6.0); + 1.0 + rng.below(5) as f64 + 10.0 / (1.0 + looseness) + }) + .collect(); + let entries = every_10s(queries.len()); + let solved = match solve(&physical, &entries, &w, &[], &[]).0 { + Ok(solved) => solved, + Err(SelectionMilpError::Infeasible) => { + report.infeasible.push(format!("{queries:?}")); + return; + } + Err(other) => panic!("{queries:?}: {other:?}"), + }; + let (selectable, _) = selection_inputs(&physical, &entries, &w, &[], &[]); + let position_of = |execution: &Execution| { + selectable + .executions() + .iter() + .position(|known| known == execution) + .unwrap() + }; + let candidate_of = |position: usize| selectable.executions()[position].candidate.0; + let mut producer_of = BTreeMap::new(); + for edge in &solved.edges { + let input = physical.edge(edge.edge).input_index_of_consumer; + producer_of.insert( + (edge.consumer.candidate.0, input), + candidate_of(position_of(&edge.producer)), + ); + } + for (query, output) in solved.outputs.iter().enumerate() { + let (_, scope) = queries[query]; + let worst = worst_union_bound(&physical, &producer_of, output.candidate.0, scope); + report.accepted += 1; + if worst > 0.01 * (1.0 + 1e-9) { + report + .violations + .push(format!("{queries:?} query {query}: worst case {worst}")); + } + } +} + +#[derive(Default)] +struct OracleReport { + accepted: usize, + infeasible: Vec, + violations: Vec, +} + +/// Every shape alone, under every scope, and paired with a per-value +/// query on `m`'s readout, each with three random draws of copies: every +/// accepted plan's union-bound worst case is within δ = 0.01. Every shape +/// planned alone that has a plan without a failure probability has one +/// with it (Pass 2's level for a query meets (K) for that query). A +/// filter or `avg by (job)` above `m or b` has none either way: no error +/// contract covers its label over two sources. +#[test] +fn every_accepted_plan_meets_its_failure_probability_on_every_dataset() { + let mut rng = Rng(1); + let mut report = OracleReport::default(); + let scopes = [ + AccuracyScope::WholeResult, + AccuracyScope::PerValue, + AccuracyScope::GroupsWithRecordShareAtLeast(0.5), + ]; + let mut without_plan = Vec::new(); + for _ in 0..3 { + for &shape in SHAPES { + for scope in scopes { + let mut alone = OracleReport::default(); + check(&[(shape, scope)], Some(0.01), &mut rng, &mut alone); + if !alone.infeasible.is_empty() { + let mut without_delta = OracleReport::default(); + check(&[(shape, scope)], None, &mut rng, &mut without_delta); + if without_delta.infeasible.is_empty() { + without_plan.extend(alone.infeasible); + } + } + report.accepted += alone.accepted; + report.violations.extend(alone.violations); + } + check( + &[ + (RM, AccuracyScope::PerValue), + (shape, AccuracyScope::WholeResult), + ], + Some(0.01), + &mut rng, + &mut report, + ); + } + } + assert!(report.violations.is_empty(), "{:#?}", report.violations); + assert!(without_plan.is_empty(), "{without_plan:#?}"); + assert!( + report.accepted >= 3 * SHAPES.len() * scopes.len(), + "{}", + report.accepted + ); +} diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs index 56b6af163..0e1f213a2 100644 --- a/crates/types/src/accuracy/target/tests.rs +++ b/crates/types/src/accuracy/target/tests.rs @@ -333,28 +333,166 @@ fn exact_targets() { ); } -/// A whole-result δ splits over the values one evaluation returns; a -/// per-value δ does not; no δ, or a whole-result one over an unknown count, -/// has none. +fn counts(per_evaluation: Option, per_output_value: Option) -> ReadoutValueCounts { + ReadoutValueCounts { + per_evaluation, + per_output_value, + } +} + +/// `q * on() group_left e`, `e` one sketched series matched onto 1000: +/// `R` = 1 is below `V · fan-in` = 1000, so every estimate read bounds the +/// whole result, at δ itself. #[test] -fn a_whole_result_failure_probability_splits_over_the_output_values() { +fn a_whole_result_reading_fewer_estimates_than_its_values_bounds_every_estimate_read() { let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + let bound = whole.failure_probability_bound(Some(1_000), counts(Some(1), Some(1))); assert_eq!( - whole.failure_probability_per_output_value(Some(100)), - Some(1e-4) + bound, + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) ); - assert_eq!(whole.failure_probability_per_output_value(None), None); - let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); assert_eq!( - per_value.failure_probability_per_output_value(None), + whole.failure_probability_per_readout_value(Some(1_000), counts(Some(1), Some(1))), Some(0.01) ); +} + +/// `q or r` over 100 series each: 200 estimates read, 100 · 1 a covered +/// value depends on, so each covered value is checked at δ / 100. +#[test] +fn an_or_over_more_estimates_than_its_values_bounds_each_covered_value() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + assert_eq!( + whole.failure_probability_bound(Some(100), counts(Some(200), Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 100, + limit: 1e-4 + }) + ); assert_eq!( - AccuracyTarget::max_error(0.01).failure_probability_per_output_value(Some(100)), + whole.failure_probability_per_readout_value(Some(100), counts(Some(200), Some(1))), + Some(1e-4) + ); +} + +/// `max(q)` over 1000 series: every estimate read, 1e-5 per readout value. +#[test] +fn a_reduction_bounds_every_estimate_read() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + let read = counts(Some(1_000), Some(1_000)); + assert_eq!( + whole.failure_probability_bound(Some(1), read), + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) + ); + assert_eq!( + whole.failure_probability_per_readout_value(Some(1), read), + Some(1e-5) + ); +} + +/// A per-value target covers one value: δ is not divided by $n_q$. +#[test] +fn a_per_value_target_covers_one_value() { + let per_value = AccuracyTarget::max_error(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + assert_eq!( + per_value.failure_probability_bound(Some(8), counts(Some(8), Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 1, + limit: 0.01 + }) + ); +} + +/// At most ⌊1 / 0.05⌋ = 20 groups hold 5% of the records: over 1000 +/// estimates each covered group is checked at 0.01 / 20; over 10, every +/// estimate read at 0.01 (1e-3 per readout value). +#[test] +fn a_record_share_covers_at_most_one_over_its_share() { + let share = AccuracyTarget::max_error(0.01) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::GroupsWithRecordShareAtLeast(0.05)); + assert_eq!( + share.failure_probability_bound(Some(1_000), counts(Some(1_000), Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 20, + limit: 5e-4 + }) + ); + assert_eq!( + share.failure_probability_bound(Some(1_000), counts(Some(10), Some(1))), + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) + ); + assert_eq!( + share.failure_probability_per_readout_value(Some(1_000), counts(Some(10), Some(1))), + Some(1e-3) + ); +} + +/// An unknown count leaves the other bound; with both unknown there is +/// none, and no target δ gives none either. +#[test] +fn an_unbound_count_leaves_the_other_bound() { + let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); + assert_eq!( + whole.failure_probability_bound(None, counts(Some(1_000), Some(1))), + Some(FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }) + ); + assert_eq!( + whole.failure_probability_bound(Some(100), counts(None, Some(1))), + Some(FailureProbabilityBound::EachCoveredValue { + values_covered: 100, + limit: 1e-4 + }) + ); + assert_eq!( + whole.failure_probability_bound(None, counts(None, Some(1))), + None + ); + assert_eq!( + AccuracyTarget::max_error(0.01) + .failure_probability_bound(Some(1), counts(Some(1), Some(1))), None ); } +/// Pass 2's per-readout split times the estimates the named bound counts +/// (`R`, or `V · fan-in`) is δ: readouts at the split meet the bound by +/// construction. +#[test] +fn the_readout_split_meets_the_bound_it_names() { + for delta in [1e-2, 1e-6] { + for values in [Some(1), Some(7), Some(100), None] { + for read in [Some(1), Some(7), Some(200), None] { + for fan_in in [Some(1), Some(7), read] { + let target = AccuracyTarget::max_error(0.01).failure_probability(delta); + let readout_values = counts(read, fan_in); + let Some(bound) = target.failure_probability_bound(values, readout_values) + else { + continue; + }; + let split = target + .failure_probability_per_readout_value(values, readout_values) + .unwrap(); + let total = match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => { + split * read.unwrap().max(1) as f64 + } + FailureProbabilityBound::EachCoveredValue { values_covered, .. } => { + split * (values_covered * fan_in.unwrap().max(1)) as f64 + } + }; + assert!( + (total - delta).abs() <= 1e-12 * delta, + "{values:?} {readout_values:?}: {total}" + ); + } + } + } + } +} + /// δ = 0.01 splits over min(R, V · fan-in) readout values; an unknown term /// drops out of the `min`. #[test] diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 1ba98db1d..9a11d59d2 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -464,6 +464,80 @@ fn max_values_per_evaluation_bounds_a_readout_by_its_states_keys() { } } +/// A top-k readout estimates every distinct item of each state it ranks: +/// over `latency` with 1000 series, 20 jobs and at most 30 distinct items +/// per evaluation declared, a heap per job estimates at most 20 × 30 = +/// 600 items (fewer than the 1000 series its rows come from), and one +/// heap over everything 30. Without the distinct bound, the series bound +/// it: 1000. A quantile estimates its values, one per state. +#[test] +fn max_estimates_per_evaluation_counts_every_item_a_top_k_ranks() { + for (max_distinct, reduction_is_by_job, estimates) in [ + (Some(30), true, 600), + (Some(30), false, 30), + (None, true, 1000), + ] { + let data = crate::workload::DataWorkload { + sources: [( + "latency".to_string(), + crate::workload::SourceFacts { + series_count: Some(1000), + distinct_values_per_label: [("job".to_string(), 20)].into(), + max_distinct_per_evaluation: max_distinct, + ..Default::default() + }, + )] + .into(), + ..Default::default() + }; + let mut memo = Memo::new(&data); + let input = memo + .find_or_add_group(scan( + "latency", + &[("job", crate::ir::schema::DataType::Utf8)], + vec![], + )) + .unwrap(); + let job = memo.groups()[input.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == "job") + .expect("the scan lists job"); + let reduction = if reduction_is_by_job { + Reduction::by(vec![job]) + } else { + Reduction::by(vec![]) + }; + let state_of = |memo: &mut Memo, algorithm| { + memo.find_or_add_group(state_grouped_by( + FieldDataType::Sketch(algorithm), + reduction.clone(), + input, + )) + .unwrap() + }; + let heap = state_of(&mut memo, SketchAlgorithm::CmsWithHeap); + let top_k = memo + .find_or_add_group(readout(SketchStatistic::TopK { k: 5 }, heap)) + .unwrap(); + assert_eq!( + memo.max_estimates_per_evaluation(top_k), + Some(estimates), + "{max_distinct:?} {reduction_is_by_job}" + ); + let kll = state_of(&mut memo, SketchAlgorithm::Kll); + let quantile = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.9 }, kll)) + .unwrap(); + assert_eq!( + memo.max_estimates_per_evaluation(quantile), + memo.max_values_per_evaluation(quantile) + ); + } +} + /// A distinct count over `events`, bounded at 9,636 distinct items per /// evaluation, counts at most 9,636 in every group above the scan; a join /// of two bounded sources counts at most their sum; a source without a From ee637c92f71c8537b9c90ca9e881d951828ad7f4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:25:15 +0000 Subject: [PATCH 3/5] fix(accuracy): constraint (K) checks the union bound over the estimates each query depends on (K) carried one failure probability D per execution, added each selected readout's delta once and checked D <= delta / V at the root. That summed both sides of `or`, divided by V where every output reads one estimate (`q * on() group_left e`), and trusted a readout level given for another query: `max(q)` over 1,000 series could read a per-value query's copy failing with 0.01 per estimate. Each execution now has P (one output value) and W (its whole output), composed by the operator classes Pass 2's split uses, over the inputs whose error reaches the output. A readout has P = delta and W = R delta (a top-k R delta for both). Each query's one row checks W <= delta (every estimate read) or P <= delta / V (each covered value), the bound AccuracyTarget::failure_probability_bound names from the same counts Pass 2 splits by; a query with several roots sums its roots' terms. - asap-types: AccuracyTarget::failure_probability_bound and FailureProbabilityBound replace failure_probability_per_output_value; the per-readout split derives from the bound. Memo::max_estimates_per_evaluation counts every item a top-k ranks. - asap-accuracy: the operator classes and the "error reaches the output" test move to estimate_dependence, shared by Pass 2 and Stage 3; readout_value_counts_of_queries exposes Pass 2's counts. - Stage 3: or and a concatenation take the largest input error, not the sum, as Pass 2's caps assume. - Stage 3: P and W exist only where a query root above checks them; created wherever bounded, RangeHeavy-8users solved about 3x slower. - accuracy-models.md: the (K) rows, which estimates a result depends on, the R assumption, examples. --- crates/accuracy/src/estimate_dependence.rs | 101 +++ crates/accuracy/src/lib.rs | 6 +- crates/accuracy/src/metric_rule/mod.rs | 15 + crates/accuracy/src/readout_caps/mod.rs | 115 ++- crates/accuracy/src/readout_caps/tests.rs | 9 +- crates/stage3-plan-selection/src/lib.rs | 4 +- .../src/milp/accuracy.rs | 669 +++++++++++++++--- .../src/milp/model/mod.rs | 27 +- crates/stage3-plan-selection/src/report.rs | 10 +- crates/types/src/accuracy/mod.rs | 3 +- crates/types/src/accuracy/target/mod.rs | 142 ++-- crates/types/src/stages/logical/memo/mod.rs | 29 + docs/design_docs/concepts/accuracy-models.md | 293 ++++++-- 13 files changed, 1113 insertions(+), 310 deletions(-) create mode 100644 crates/accuracy/src/estimate_dependence.rs diff --git a/crates/accuracy/src/estimate_dependence.rs b/crates/accuracy/src/estimate_dependence.rs new file mode 100644 index 000000000..5150e30c4 --- /dev/null +++ b/crates/accuracy/src/estimate_dependence.rs @@ -0,0 +1,101 @@ +//! Which estimates an output value depends on (`accuracy-models.md`, +//! "Which estimates a result depends on"): the operator classes and the +//! "error reaches the output" test that Pass 2's failure-probability split +//! (`readout_caps`) and Stage 3's constraint (K) both count by, so the +//! level Pass 2 gives a query meets (K) for that query. +//! +//! An estimate is one sketch readout value in one evaluation. An output +//! value depends on one value of each input an operator reads value by +//! value ([`inputs_read_value_by_value`]), on one input's value when the +//! operator copies each value from one input +//! ([`takes_each_value_from_one_input`]), and on every value of any other +//! input; only through inputs whose error reaches the output +//! ([`input_error_reaches_output`], which Stage 3 reads through +//! `ErrorMetricsOfMemo::input_error_reaches_output`). + +use asap_types::ir::operator::arguments::BinaryOpKind; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; +use asap_types::stages::MemoGroupId; +use asap_types::summary_type::SketchStatistic; + +use crate::metric_rule::ErrorContractPerMetric; + +/// Whether input `input` of a candidate with the error contracts +/// `contract_per_metric` can make its output miss: its error reaches the +/// output in some metric ($\beta > 0$). A count's input and the right +/// side of `and` and `unless` cannot ($\beta = 0$: only which series or +/// rows they hold is read), nor any input of an operator with no contract +/// (a subquery's operand, which may not carry error at all). +pub(crate) fn input_error_reaches_output( + contract_per_metric: &ErrorContractPerMetric, + input: usize, +) -> bool { + contract_per_metric + .iter() + .flatten() + .any(|contract| contract.input_error_reaches_output(input)) +} + +/// Whether `operator`, a readout, chooses its output values by the +/// estimates it computes: a top-k ranks every item of its state, so each +/// of its output values depends on all of them. +pub fn ranks_by_estimate(operator: &Operator) -> bool { + matches!( + operator, + Operator::ASAP(ASAPOp::SummaryEstimate { + query: SketchStatistic::TopK { .. }, + .. + }) + ) +} + +/// How many direct inputs `operator` reads value by value: each output +/// value from one value (row or series) of each of them; `None` for an +/// operator one of whose output values may read many values of an input +/// (an aggregate, a window function, a time range or subquery, a `limit` +/// choosing rows by value, a deduplication). A filter, projection, sort, +/// time shift, relabel, label enrichment or series sample: 1, each output +/// value one input value. A binary operator, join or relational set +/// operator: 2, one value of each side; a concatenation: each of its +/// inputs. Operators its scalar expressions reference (`scalar(r)`) follow +/// the direct inputs and are read whole. Both counts are per evaluation, +/// not per subquery step: `max_over_time(q[1h:1m])` reads each readout +/// value of `q` at 60 steps, which they do not multiply in (no cap passes +/// through a subquery to the readouts below it). +pub fn inputs_read_value_by_value(operator: &Operator) -> Option { + match operator { + Operator::NonASAP( + NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. }, + ) => Some(1), + Operator::NonASAP( + NonASAPOp::BinaryOp { .. } | NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. }, + ) => Some(2), + Operator::NonASAP(NonASAPOp::Concat { children }) => Some(children.len()), + _ => None, + } +} + +/// Whether `operator` copies each output value from exactly one of its +/// inputs: a PromQL set operator (`q or r` takes a series' value from `q` +/// when `q` has the series, else from `r`; `and` and `unless` only from +/// the left) or a concatenation (each row from one of its inputs). +pub fn takes_each_value_from_one_input(operator: &Operator) -> bool { + matches!( + operator, + Operator::NonASAP( + NonASAPOp::BinaryOp { + operator: BinaryOperator { + kind: BinaryOpKind::Set(_), + .. + }, + .. + } | NonASAPOp::Concat { .. } + ) + ) +} diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 075ac794c..6c78f3a4c 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -18,6 +18,7 @@ //! can use it. mod error_contract; +mod estimate_dependence; mod evidence; mod guarantee; mod logical_edge; @@ -27,6 +28,9 @@ mod readout; mod readout_caps; pub use error_contract::ErrorContract; +pub use estimate_dependence::{ + inputs_read_value_by_value, ranks_by_estimate, takes_each_value_from_one_input, +}; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::{tolerated_limit, within_limit}; @@ -35,4 +39,4 @@ pub use measured::{ MeasuredSketchAccuracyError, }; pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; -pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; +pub use readout_caps::{readout_levels, readout_value_counts_of_queries, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 6e947e0e4..ce8dbdd7f 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -104,6 +104,21 @@ impl ErrorMetricsOfMemo { .as_ref() } + /// Whether `candidate`'s input `input` can make its output miss: its + /// error reaches the output in some metric + /// ([`crate::estimate_dependence::input_error_reaches_output`], the + /// test Pass 2's failure-probability split counts by). + pub fn input_error_reaches_output( + &self, + candidate: CandidateForMemoGroupId, + input: usize, + ) -> bool { + crate::estimate_dependence::input_error_reaches_output( + &self.error_contract[candidate.group.0 as usize][candidate.index_in_group], + input, + ) + } + /// The error metric of `candidate` when it is a sketch readout. pub(crate) fn readout_error_metric( &self, diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index 6ada0ba8d..de3c1329a 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -6,12 +6,15 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use asap_types::accuracy::{AccuracyMetric, AccuracyTarget, ReadoutValueCounts}; -use asap_types::ir::operator::arguments::BinaryOpKind; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; +use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId, RequiredAccuracy}; -use asap_types::summary_type::{parameter_list, SketchStatistic}; +use asap_types::summary_type::parameter_list; +use crate::estimate_dependence::{ + input_error_reaches_output, inputs_read_value_by_value, ranks_by_estimate, + takes_each_value_from_one_input, +}; use crate::logical_edge::build_meets_level; use crate::metric_rule::{query_roots_with_target, ErrorContractPerMetric}; use crate::readout::sketch_readout; @@ -113,10 +116,7 @@ pub fn readout_levels( } } } - let readout_groups: BTreeSet = - readouts.iter().map(|(readout, _)| readout.group).collect(); - let readouts_read = - readouts_read_by_queries(memo, &error_metrics.error_contract, order, &readout_groups); + let readouts_read = readouts_read_by_queries(memo, error_metrics, order); let mut readouts_of_group: HashMap< (MemoGroupId, AccuracyMetric), Vec, @@ -430,10 +430,26 @@ struct ReadoutsReadByQueries { /// memo's topological order) from the readout groups `readout_groups`. fn readouts_read_by_queries( memo: &Memo, - error_contract: &[Vec], + error_metrics: &ErrorMetricsOfMemo, order: &[MemoGroupId], - readout_groups: &BTreeSet, ) -> ReadoutsReadByQueries { + let error_contract = &error_metrics.error_contract; + let readout_groups: BTreeSet = memo + .groups() + .iter() + .enumerate() + .filter(|(group, memo_group)| { + (0..memo_group.candidates.len()).any(|index_in_group| { + error_metrics + .readout_error_metric(CandidateForMemoGroupId { + group: MemoGroupId(*group as u32), + index_in_group, + }) + .is_some() + }) + }) + .map(|(group, _)| MemoGroupId(group as u32)) + .collect(); // Per group: the readout values its output can depend on (`Some(0)` // when none, so it carries no readout error; `None` unknown). let none_read = ReadoutValueCounts { @@ -454,10 +470,7 @@ fn readouts_read_by_queries( let takes_each_value_from_one_input = takes_each_value_from_one_input(operator); let reads_with_error = |input: &usize| { readout_values_per_group[children[*input].0 as usize].per_evaluation != Some(0) - && contract_per_metric - .iter() - .flatten() - .any(|contract| contract.error_factor_per_input[*input] > 0.0) + && input_error_reaches_output(contract_per_metric, *input) }; let inputs_reading_with_error = (0..children.len()).filter(reads_with_error).count(); // Each output value comes from one input, so the output's error @@ -510,16 +523,10 @@ fn readouts_read_by_queries( // output keys are chosen by their estimates; any other candidate of // the group (a reduction over readouts below it) still counts. readout_values_per_group[group.0 as usize] = if readout_groups.contains(&group) { - let per_evaluation = memo.max_values_per_evaluation(group); - let ranks_by_estimate = candidates.iter().any(|candidate| { - matches!( - candidate.expression.operator, - Operator::ASAP(ASAPOp::SummaryEstimate { - query: SketchStatistic::TopK { .. }, - .. - }) - ) - }); + let per_evaluation = memo.max_estimates_per_evaluation(group); + let ranks_by_estimate = candidates + .iter() + .any(|candidate| ranks_by_estimate(&candidate.expression.operator)); let own = ReadoutValueCounts { per_evaluation, per_output_value: if ranks_by_estimate { @@ -546,55 +553,19 @@ fn readouts_read_by_queries( } } -/// How many direct inputs `operator` reads value by value: each output -/// value from one value (row or series) of each of them; `None` for an -/// operator one of whose output values may read many values of an input -/// (an aggregate, a window function, a time range or subquery, a `limit` -/// choosing rows by value, a deduplication). A filter, projection, sort, -/// time shift, relabel, label enrichment or series sample: 1, each output -/// value one input value. A binary operator, join or relational set -/// operator: 2, one value of each side; a concatenation: each of its -/// inputs. Operators its scalar expressions reference (`scalar(r)`) follow -/// the direct inputs and are read whole. Both counts are per evaluation, -/// not per subquery step: `max_over_time(q[1h:1m])` reads each readout -/// value of `q` at 60 steps, which they do not multiply in (no cap passes -/// through a subquery to the readouts below it). -fn inputs_read_value_by_value(operator: &Operator) -> Option { - match operator { - Operator::NonASAP( - NonASAPOp::Filter { .. } - | NonASAPOp::Project { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::TimeShift { .. } - | NonASAPOp::PromqlRelabel { .. } - | NonASAPOp::PromqlInfoEnrich { .. } - | NonASAPOp::PromqlSeriesSample { .. }, - ) => Some(1), - Operator::NonASAP( - NonASAPOp::BinaryOp { .. } | NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. }, - ) => Some(2), - Operator::NonASAP(NonASAPOp::Concat { children }) => Some(children.len()), - _ => None, - } -} - -/// Whether `operator` copies each output value from exactly one of its -/// inputs: a PromQL set operator (`q or r` takes a series' value from `q` -/// when `q` has the series, else from `r`; `and` and `unless` only from -/// the left) or a concatenation (each row from one of its inputs). -fn takes_each_value_from_one_input(operator: &Operator) -> bool { - matches!( - operator, - Operator::NonASAP( - NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Set(_), - .. - }, - .. - } | NonASAPOp::Concat { .. } - ) - ) +/// Per query with a target, by index: the readout values (estimates) its +/// output can depend on per evaluation, `R` and the fan-in +/// ([`ReadoutsReadByQueries::readout_values`]), which +/// `AccuracyTarget::failure_probability_bound` chooses the query's bound +/// from. Pass 2 splits δ by it, and Stage 3's constraint (K) checks the +/// same bound, so the two cannot choose differently. `order` is the memo's +/// topological order (`Memo::topological_order`). +pub fn readout_value_counts_of_queries( + memo: &Memo, + error_metrics: &ErrorMetricsOfMemo, + order: &[MemoGroupId], +) -> BTreeMap { + readouts_read_by_queries(memo, error_metrics, order).readout_values } /// Keeps the larger of `cap` and the known cap of `query` in `metric`. diff --git a/crates/accuracy/src/readout_caps/tests.rs b/crates/accuracy/src/readout_caps/tests.rs index 47f49a058..bcd0ec73e 100644 --- a/crates/accuracy/src/readout_caps/tests.rs +++ b/crates/accuracy/src/readout_caps/tests.rs @@ -4,7 +4,7 @@ use asap_types::accuracy::AccuracyScope; use asap_types::ir::operator::arguments::{PromQLVectorSetOpKind, Reduction}; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::QueryRoot; +use asap_types::ir::{NonASAPOp, QueryRoot}; use asap_types::stages::{MemoQuery, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use asap_types::test_support::memo_builders::{ @@ -616,12 +616,7 @@ fn a_per_value_ratio_splits_over_the_two_readout_values_of_each_output_value() { with_query(&mut memo, ratio, target.clone(), Some(8)); let order = memo.topological_order().expect("acyclic"); let error_metrics = ErrorMetricsOfMemo::of(&memo, &order); - let read = readouts_read_by_queries( - &memo, - &error_metrics.error_contract, - &order, - &[p90, p50].into(), - ); + let read = readouts_read_by_queries(&memo, &error_metrics, &order); assert!(read.composed.contains(&0)); let counts = read.readout_values[&0]; assert_eq!( diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index d936a81b8..47d3579b2 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -128,10 +128,10 @@ pub fn select_plan( SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_per_output_value, + failure_probability_bound, } => ErrorSelectingPlan::AccuracyTargetBelowSolverPrecision { query: query.0, - failure_probability_per_output_value, + failure_probability_bound, }, })?; let plan = plan_of(physical, &solved, &outputs).map_err(|error| { diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index 5f530e122..e6d183a67 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -14,8 +14,12 @@ use std::hash::{Hash, Hasher}; use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; -use asap_accuracy::{query_roots_with_target, tolerated_limit, ErrorContract, ErrorMetricsOfMemo}; -use asap_types::accuracy::AccuracyMetric; +use asap_accuracy::{ + inputs_read_value_by_value, query_roots_with_target, ranks_by_estimate, + readout_value_counts_of_queries, takes_each_value_from_one_input, tolerated_limit, + ErrorContract, ErrorMetricsOfMemo, +}; +use asap_types::accuracy::{AccuracyMetric, FailureProbabilityBound}; use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId}; use super::equivalent_contexts::SharedVariables; @@ -23,20 +27,54 @@ use super::model::SelectionInputs; use crate::selectable::QueryId; /// An execution's error as (K) composes it: its error bound per metric -/// (by [`AccuracyMetric::index`]) and its failure probability. +/// (by [`AccuracyMetric::index`]) and two failure probabilities, each the +/// summed δ of estimates (one readout value in one evaluation) by the +/// union bound (`accuracy-models.md`, "Which estimates a result depends +/// on"). A failure probability is `None` when no declared fact bounds it +/// (a readout whose values per evaluation are unknown). /// -/// Example: a DDSketch readout required at relative 0.01 with δ = 1e-3 -/// is 0.01 in `Relative`, 0 in every other metric, and 1e-3. -#[derive(Debug, Clone, Copy, Default, PartialEq)] +/// Example: a DDSketch readout over 1000 series required at relative 0.01 +/// with δ = 1e-3 is 0.01 in `Relative`, 0 in every other metric, 1e-3 for +/// one value and 1000 × 1e-3 = 1 for all of them. +#[derive(Debug, Clone, Copy, PartialEq)] pub(super) struct ExecutionError { /// $E_{c,m}$ per metric. error_bound_per_metric: [f64; AccuracyMetric::COUNT], - /// $D_c$. - failure_probability: f64, + /// $P_c$: the largest, over its output values, of the summed δ of the + /// estimates one value depends on. + failure_probability_of_one_value: Option, + /// $W_c$: the summed δ of every estimate any of its output values + /// depends on. + failure_probability_of_all_values: Option, +} + +impl Default for ExecutionError { + /// No error, and no estimate to fail: an exact execution's. + fn default() -> Self { + Self { + error_bound_per_metric: [0.0; AccuracyMetric::COUNT], + failure_probability_of_one_value: Some(0.0), + failure_probability_of_all_values: Some(0.0), + } + } +} + +/// Sums of failure probabilities, `None` (unbounded) when any term is. +fn sum_of(terms: impl IntoIterator>) -> Option { + terms + .into_iter() + .try_fold(0.0, |total, term| Some(total + term?)) +} + +/// The largest of failure probabilities, `None` (unbounded) when any is. +fn largest_of(terms: impl IntoIterator>) -> Option { + terms + .into_iter() + .try_fold(0.0, |largest: f64, term| Some(largest.max(term?))) } impl ExecutionError { - /// The larger of `self` and `other` in each metric and in failure + /// The larger of `self` and `other` in each metric and in each failure /// probability, as the most any one of several producers passes on. fn largest(self, other: Self) -> Self { let mut largest = self; @@ -47,13 +85,58 @@ impl ExecutionError { { *error = error.max(other); } - largest.failure_probability = largest.failure_probability.max(other.failure_probability); + largest.failure_probability_of_one_value = largest_of([ + self.failure_probability_of_one_value, + other.failure_probability_of_one_value, + ]); + largest.failure_probability_of_all_values = largest_of([ + self.failure_probability_of_all_values, + other.failure_probability_of_all_values, + ]); largest } - /// Whether it is 0 in every metric and in failure probability. + /// Whether it is 0 in every metric and in each failure probability. fn is_zero(&self) -> bool { - self.failure_probability == 0.0 && self.error_bound_per_metric.iter().all(|&e| e == 0.0) + self.failure_probability_of_one_value == Some(0.0) + && self.failure_probability_of_all_values == Some(0.0) + && self.error_bound_per_metric.iter().all(|&e| e == 0.0) + } + + /// The failure probability `bound` checks: of all values for every + /// estimate read, of one value for each covered value. + fn failure_probability_checked_by(&self, bound: &FailureProbabilityBound) -> Option { + match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => { + self.failure_probability_of_all_values + } + FailureProbabilityBound::EachCoveredValue { .. } => { + self.failure_probability_of_one_value + } + } + } +} + +/// How an execution's output values read its inputs, which decides how +/// (K) composes its failure probabilities (`asap_accuracy`'s operator +/// classes, the ones Pass 2's split counts by). +#[derive(Debug, Clone, Copy)] +struct HowInputsAreRead { + /// Its first so many inputs are read value by value: each output value + /// reads one value of each ([`inputs_read_value_by_value`]); every + /// other input is read whole. `None`: all are read whole (a reduction). + value_by_value: Option, + /// Each output value is copied from one input (`or`, a concatenation: + /// [`takes_each_value_from_one_input`]), so the inputs read value by + /// value add the largest of their one-value failure probabilities, not + /// their sum. + each_value_from_one_input: bool, +} + +impl HowInputsAreRead { + /// Whether input `input` is read value by value. + fn reads_value_by_value(&self, input: usize) -> bool { + self.value_by_value.is_some_and(|count| input < count) } } @@ -61,7 +144,8 @@ impl ExecutionError { /// ([`AccuracyRules::compose`]). struct Composition { /// Its own error plus each input's error times the contract's factor, - /// and its own failure probability plus each input's. + /// and its own failure probabilities plus its inputs' by the operator + /// classes. error: ExecutionError, /// Whether every input's error passes the execution's metric rule (it /// has a contract in that metric) and the contract's validity cap. @@ -80,8 +164,13 @@ pub(super) struct AccuracyRules { /// its logical candidate. candidate_of_execution: Vec, /// Per execution, the error it adds: a readout's requirement - /// (`CandidateForMemoGroup::required_accuracy`), 0 for any other. + /// (`CandidateForMemoGroup::required_accuracy`), its δ for one value + /// (`R` times it for a top-k, which ranks every estimate) and `R` + /// times its δ for all values (`Memo::max_estimates_per_evaluation`); + /// 0 for any other. own_error_per_execution: Vec, + /// Per execution, how its output values read its inputs. + how_inputs_are_read: Vec, /// Per execution, whether a query with an accuracy target reaches its /// group. reached_by_query_with_target: Vec, @@ -100,28 +189,37 @@ pub(super) struct AccuracyRules { /// and hash by the limits' bits. /// /// Example: `p99 / p50` at relative 0.03 with δ = 0.01 over a whole result -/// of 100 values allows 0.03 (widened) in `Relative`, 0 in every other -/// metric, and 1e-4 (widened) failure probability per output value. +/// of 100 values, reading two readouts over 100 series each, allows 0.03 +/// (widened) in `Relative`, 0 in every other metric, and checks each +/// covered value: its one-value failure probability at most 1e-4 +/// (widened). #[derive(Debug, Clone, Copy)] pub(super) struct RootLimits { /// Per metric (by [`AccuracyMetric::index`]), the error the output may /// have: the target's ε in a metric the root allows /// ([`query_roots_with_target`]), 0 in any other. tolerated_error_per_metric: [f64; AccuracyMetric::COUNT], - /// The failure probability each output value may have - /// (`AccuracyTarget::failure_probability_per_output_value`); `None` - /// without a δ, or with one over the whole result and $n_q$ unknown - /// (Pass 2 then gives that query's readouts no level). - tolerated_failure_probability_per_output_value: Option, + /// The union bound the query's failure probability is checked by, with + /// its limit (`AccuracyTarget::failure_probability_bound`, widened by + /// [`tolerated_limit`] where it is compared; + /// from the readout counts Pass 2 splits δ by); `None` without a δ, + /// or with neither bound known (Pass 2 then gives the query's readouts + /// no level, and (X) makes it read them exactly). + failure_probability_bound: Option, } impl RootLimits { /// The limits' bits, which equality, hashing and order compare. - fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option) { + fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option<(u64, u64)>) { ( self.tolerated_error_per_metric.map(f64::to_bits), - self.tolerated_failure_probability_per_output_value - .map(f64::to_bits), + self.failure_probability_bound.map(|bound| match bound { + FailureProbabilityBound::EveryEstimateRead { limit } => (0, limit.to_bits()), + FailureProbabilityBound::EachCoveredValue { + values_covered, + limit, + } => (values_covered.max(1), limit.to_bits()), + }), ) } } @@ -181,13 +279,34 @@ impl AccuracyRules { .iter() .map(|&id| { let mut own = ExecutionError::default(); - if let Some(required) = memo.candidate(id).required_accuracy() { + let candidate = memo.candidate(id); + if let Some(required) = candidate.required_accuracy() { own.error_bound_per_metric[required.metric.index()] = required.max_error; - own.failure_probability = required.failure_probability; + let delta = required.failure_probability; + let of_every_estimate = memo + .max_estimates_per_evaluation(id.group) + .map(|estimates| estimates as f64 * delta); + own.failure_probability_of_one_value = + if ranks_by_estimate(&candidate.expression.operator) { + of_every_estimate + } else { + Some(delta) + }; + own.failure_probability_of_all_values = of_every_estimate; } own }) .collect(); + let how_inputs_are_read = candidate_of_execution + .iter() + .map(|&id| { + let operator = &memo.candidate(id).expression.operator; + HowInputsAreRead { + value_by_value: inputs_read_value_by_value(operator), + each_value_from_one_input: takes_each_value_from_one_input(operator), + } + }) + .collect(); let reached_by_query_with_target = candidate_of_execution .iter() .map(|id| { @@ -205,28 +324,33 @@ impl AccuracyRules { (0..candidate_of_execution.len()).collect(); executions_producers_first .sort_by_key(|&c| position_in_order[candidate_of_execution[c].group.0 as usize]); + let error_metrics = ErrorMetricsOfMemo::of(memo, &order); + let readout_values = readout_value_counts_of_queries(memo, &error_metrics, &order); let roots = query_roots_with_target(memo) .into_iter() .map(|(query, root, target, allowed_metrics)| { - let failure_probability = target.failure_probability_per_output_value( - memo.queries()[query].number_of_values_per_query_evaluation, - ); + let failure_probability_bound = readout_values.get(&query).and_then(|&counts| { + target.failure_probability_bound( + memo.queries()[query].number_of_values_per_query_evaluation, + counts, + ) + }); let mut tolerated_error_per_metric = [0.0; AccuracyMetric::COUNT]; for metric in allowed_metrics { tolerated_error_per_metric[metric.index()] = tolerated_limit(target.max_error); } let limits = RootLimits { tolerated_error_per_metric, - tolerated_failure_probability_per_output_value: failure_probability - .map(tolerated_limit), + failure_probability_bound, }; ((QueryId(query), root), limits) }) .collect(); Some(Self { - error_metrics: ErrorMetricsOfMemo::of(memo, &order), + error_metrics, candidate_of_execution, own_error_per_execution, + how_inputs_are_read, reached_by_query_with_target, executions_producers_first, roots, @@ -240,6 +364,18 @@ impl AccuracyRules { self.roots.get(&(query, group)) } + /// The union bound `query`'s failure probability is checked by, with its + /// limit; `None` without one. + pub(super) fn failure_probability_bound( + &self, + query: QueryId, + ) -> Option { + self.roots + .iter() + .find(|((root_query, _), _)| *root_query == query) + .and_then(|(_, limits)| limits.failure_probability_bound) + } + /// Execution `c`'s error contract in metric `m` (by /// [`AccuracyMetric::index`]); `None` where it has none (the metric /// rule). @@ -253,28 +389,56 @@ impl AccuracyRules { /// (`accuracy-models.md`, "Checking each query's target in /// selection"): its own error /// plus, per metric, $\beta_{c,i,m}$ times each input's error in that - /// metric where the input's error reaches the output ($\beta > 0$), - /// and its own failure probability plus every input's (a union bound). + /// metric where the input's error reaches the output ($\beta > 0$); the + /// largest of those instead when each output value is copied from one + /// input (`or`, a concatenation), whose error is that input's. Its + /// failure probabilities add, over the inputs whose error reaches + /// the output in some metric (a union bound over estimates): + /// - of one value, its own plus the one-value failure probability of + /// each input read value by value (the largest of them when each + /// value is copied from one input) plus the all-values one of each + /// input read whole; + /// - of all values, its own plus each input's all-values one. + /// /// An input's error in a metric `c` has no contract for breaks the /// metric rule, and one above the contract's validity cap $b_{c,i}$ /// breaks the cap: either leaves the inputs not admitted, and adds /// nothing in that metric for the metric rule. /// - /// Example: `p99 / p50`'s contract passes each operand's relative + /// Examples: `p99 / p50`'s contract passes each operand's relative /// error with factor 1.111 and caps the denominator at 0.1: reading /// 0.02 and 0.02 composes 0.044, admitted; reading a denominator at - /// 0.2 is not admitted. + /// 0.2 is not admitted. `max` over a readout of 1000 series at δ 1e-5 + /// reads it whole: 0.01 for one value and for all. `q or r` over + /// readouts at δ 1e-4 of 100 series each: 1e-4 for one value, 0.02 + /// for all. fn compose( &self, c: usize, inputs_read: impl IntoIterator, ) -> Composition { + let own = self.own_error_per_execution[c]; let mut composed = Composition { - error: self.own_error_per_execution[c], + error: own, inputs_admitted: true, }; + let how = self.how_inputs_are_read[c]; + let mut one_value_of_inputs_read_value_by_value = Vec::new(); + let mut all_values_of_inputs_read_whole = Vec::new(); + let mut all_values_of_inputs = Vec::new(); for (i, read) in inputs_read { - composed.error.failure_probability += read.failure_probability; + if self + .error_metrics + .input_error_reaches_output(self.candidate_of_execution[c], i) + { + all_values_of_inputs.push(read.failure_probability_of_all_values); + if how.reads_value_by_value(i) { + one_value_of_inputs_read_value_by_value + .push(read.failure_probability_of_one_value); + } else { + all_values_of_inputs_read_whole.push(read.failure_probability_of_all_values); + } + } for (m, &error) in read.error_bound_per_metric.iter().enumerate() { if error <= 0.0 { continue; @@ -291,10 +455,30 @@ impl AccuracyRules { { composed.inputs_admitted = false; } - composed.error.error_bound_per_metric[m] += - contract.error_factor_per_input[i] * error; + let of_input = contract.error_factor_per_input[i] * error; + let composed_error = &mut composed.error.error_bound_per_metric[m]; + if how.each_value_from_one_input { + // Each output value's error is one input's. + *composed_error = composed_error.max(own.error_bound_per_metric[m] + of_input); + } else { + *composed_error += of_input; + } } } + let of_inputs_read_value_by_value = if how.each_value_from_one_input { + largest_of(one_value_of_inputs_read_value_by_value) + } else { + sum_of(one_value_of_inputs_read_value_by_value) + }; + composed.error.failure_probability_of_one_value = sum_of([ + own.failure_probability_of_one_value, + of_inputs_read_value_by_value, + sum_of(all_values_of_inputs_read_whole), + ]); + composed.error.failure_probability_of_all_values = sum_of([ + own.failure_probability_of_all_values, + sum_of(all_values_of_inputs), + ]); composed } @@ -304,8 +488,11 @@ impl AccuracyRules { /// meets (K), exactly in f64: each execution a query with a target /// reaches, composed ([`Self::compose`]) from the producers selected /// on its inputs, admits its inputs' errors and, at each output of a - /// query with a target, is within the root's limits. `Err` names the - /// first query (in `answering` order) whose output misses its target. + /// query with a target, is within the root's error limits; and each + /// query's failure probability its bound checks, summed over its + /// outputs (a query with several roots reads them all), is within its + /// limit. `Err` names the first query (in `answering` order) whose + /// output misses its target. /// /// Example: p99 / p50 at relative 0.03 over two readouts at 0.02 does /// not meet it (1.111 · 0.04 = 0.044), so it is `Err` with the ratio's @@ -327,6 +514,11 @@ impl AccuracyRules { let mut composed: Vec> = vec![None; count]; let mut on_walk = vec![false; count]; let memo = &inputs.physical.logical_candidates.memo; + // Per query with a δ, in `answering` order: its bound and the + // failure probability it checks, summed over the outputs so far + // (`None` once one is unbounded). + let mut failure_probability_of_query: Vec<(QueryId, FailureProbabilityBound, Option)> = + Vec::new(); for (&(_, group), &(query, v)) in memo.output_groups().iter().zip(answering) { // Compose `v` and every execution below it, producers first. let mut pending: Vec<(usize, bool)> = vec![(v, false)]; @@ -377,12 +569,24 @@ impl AccuracyRules { .iter() .zip(limits.tolerated_error_per_metric) .any(|(&error, limit)| error > limit) - || limits - .tolerated_failure_probability_per_output_value - .is_some_and(|limit| error.failure_probability > limit) { return Err(query); } + if let Some(bound) = limits.failure_probability_bound { + let checked = error.failure_probability_checked_by(&bound); + match failure_probability_of_query + .iter_mut() + .find(|(known, _, _)| *known == query) + { + Some((_, _, total)) => *total = sum_of([*total, checked]), + None => failure_probability_of_query.push((query, bound, checked)), + } + } + } + for (query, bound, total) in failure_probability_of_query { + if total.is_none_or(|total| total > tolerated_limit(bound.limit())) { + return Err(query); + } } Ok(()) } @@ -414,17 +618,25 @@ pub(super) struct ErrorModel<'a> { /// $y$: (position in `SelectableExecutions::edges`, producer /// execution); empty for an execution without rows. selectable_edges_per_input: Vec>>, - /// $M$: the largest error and failure probability any selection gives - /// the execution. + /// $M$ (and $M^P$, $M^W$): the largest error and failure probabilities + /// any selection gives the execution; a failure probability is `None` + /// when some selection leaves it unbounded. ceiling: Vec, + /// Whether some query root above the execution checks its failure + /// probability of one value ($P$) and of all values ($W$), through the + /// rows it reads them by: a root checking each covered value reads $P$, + /// one checking every estimate read $W$; $P$ reads $P$ of an input read + /// value by value and $W$ of one read whole, $W$ reads $W$. (K) has a + /// variable only for a failure probability some root checks. + checked_failure_probabilities: Vec<(bool, bool)>, /// Whether the execution's error is constant: its inputs carry none /// under any selection, so its ceilings are its own error. error_is_constant: Vec, } -/// An execution's error in one metric, or its failure probability, as -/// (K)'s rows read it: a constant (a readout's -/// own requirement) or a variable with its ceiling. +/// An execution's error in one metric, or one of its failure +/// probabilities, as (K)'s rows read it: a constant (a readout's own +/// requirement) or a variable with its ceiling. #[derive(Clone, Copy)] enum ErrorTerm { Constant(f64), @@ -468,9 +680,11 @@ impl<'a> ErrorModel<'a> { selectable_edges_per_input: (0..count).map(selectable_edges_of).collect(), has_rows, ceiling: vec![ExecutionError::default(); count], + checked_failure_probabilities: vec![(false, false); count], error_is_constant: vec![false; count], }; model.set_ceilings(); + model.set_checked_failure_probabilities(inputs, shared); model.error_is_constant = (0..count) .map(|c| { model.has_rows[c] @@ -497,17 +711,124 @@ impl<'a> ErrorModel<'a> { } } - /// Execution `u`'s failure probability; `None` when it is 0 - /// under every selection. - fn failure_probability(&self, accuracy: &AccuracyVariables, u: usize) -> Option { - let ceiling = self.ceiling[u].failure_probability; - match (ceiling > 0.0, self.error_is_constant[u]) { + /// Execution `u`'s failure probability of one value ($P_u$); `Ok(None)` + /// when it is 0 under every selection, `Err` when some selection + /// leaves it unbounded. + fn failure_probability_of_one_value( + &self, + accuracy: &AccuracyVariables, + u: usize, + ) -> Result, Unbounded> { + self.failure_probability_term( + self.ceiling[u].failure_probability_of_one_value, + u, + &accuracy.failure_probability_of_one_value, + ) + } + + /// Execution `u`'s failure probability of all values ($W_u$), as + /// [`Self::failure_probability_of_one_value`]. + fn failure_probability_of_all_values( + &self, + accuracy: &AccuracyVariables, + u: usize, + ) -> Result, Unbounded> { + self.failure_probability_term( + self.ceiling[u].failure_probability_of_all_values, + u, + &accuracy.failure_probability_of_all_values, + ) + } + + /// The failure probability `variables` holds for `u`, whose ceiling is + /// `ceiling`, as a term. + fn failure_probability_term( + &self, + ceiling: Option, + u: usize, + variables: &HashMap, + ) -> Result, Unbounded> { + let ceiling = ceiling.ok_or(Unbounded)?; + Ok(match (ceiling > 0.0, self.error_is_constant[u]) { (false, _) => None, (true, true) => Some(ErrorTerm::Constant(ceiling)), (true, false) => Some(ErrorTerm::Variable { - variable: accuracy.failure_probability[&u], + variable: variables[&u], ceiling, }), + }) + } + + /// The failure probability `bound` checks of execution `v` (of all + /// values for every estimate read, of one value for each covered + /// value). + fn failure_probability_checked_by( + &self, + accuracy: &AccuracyVariables, + v: usize, + bound: &FailureProbabilityBound, + ) -> Result, Unbounded> { + match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => { + self.failure_probability_of_all_values(accuracy, v) + } + FailureProbabilityBound::EachCoveredValue { .. } => { + self.failure_probability_of_one_value(accuracy, v) + } + } + } + + /// [`Self::checked_failure_probabilities`], in one pass from each + /// original query's answering executions down, consumers first. + fn set_checked_failure_probabilities( + &mut self, + inputs: &SelectionInputs<'_>, + shared: &SharedVariables, + ) { + let rules = self.rules; + let memo = &inputs.physical.logical_candidates.memo; + for (&(query, group), (query_id, executions)) in memo + .output_groups() + .iter() + .zip(inputs.selectable.answering()) + { + if shared.original_of_query[query] != *query_id { + continue; + } + let Some(bound) = rules + .root_limits(*query_id, group) + .and_then(|limits| limits.failure_probability_bound) + else { + continue; + }; + for &v in executions { + let checked = &mut self.checked_failure_probabilities[v]; + match bound { + FailureProbabilityBound::EveryEstimateRead { .. } => checked.1 = true, + FailureProbabilityBound::EachCoveredValue { .. } => checked.0 = true, + } + } + } + for &c in rules.executions_producers_first.iter().rev() { + let (one_value, all_values) = self.checked_failure_probabilities[c]; + if !self.has_rows[c] || !(one_value || all_values) { + continue; + } + let how = rules.how_inputs_are_read[c]; + for (i, edges) in self.selectable_edges_per_input[c].iter().enumerate() { + if !rules + .error_metrics + .input_error_reaches_output(rules.candidate_of_execution[c], i) + { + continue; + } + let value_by_value = how.reads_value_by_value(i); + for &(_, u) in edges { + let checked = &mut self.checked_failure_probabilities[u]; + checked.0 |= one_value && value_by_value; + checked.1 |= all_values || (one_value && !value_by_value); + } + } } } @@ -549,24 +870,35 @@ impl<'a> ErrorModel<'a> { } } +/// A failure probability no declared fact bounds under some selection +/// (a readout whose values per evaluation are unknown, below): (K) has no +/// variable for it, and no output may be answered by a bound reading it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Unbounded; + /// The accuracy variables of constraint (K), all continuous, in /// each only for an execution with rows /// ([`ErrorModel`]) whose error is not constant and whose ceiling is above -/// 0, by execution position. +/// 0 (and, for a failure probability, bounded and checked by some query +/// root above: [`ErrorModel::checked_failure_probabilities`]), by +/// execution position. pub(super) struct AccuracyVariables { /// $E_{\kappa,c,m}$: the execution's error bound in metric $m$ (by /// [`AccuracyMetric::index`]). error_bound: HashMap<(usize, usize), Variable>, - /// $D_{\kappa,c}$: its failure probability. - failure_probability: HashMap, + /// $P_{\kappa,c}$: its failure probability of one value. + failure_probability_of_one_value: HashMap, + /// $W_{\kappa,c}$: its failure probability of all values. + failure_probability_of_all_values: HashMap, } impl AccuracyVariables { - /// $E$ and $D$ over `model`, added to `problem`. + /// $E$, $P$ and $W$ over `model`, added to `problem`. pub(super) fn new(model: &ErrorModel<'_>, problem: &mut ProblemVariables) -> Self { let mut variables = Self { error_bound: HashMap::new(), - failure_probability: HashMap::new(), + failure_probability_of_one_value: HashMap::new(), + failure_probability_of_all_values: HashMap::new(), }; for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) @@ -578,47 +910,74 @@ impl AccuracyVariables { .insert((c, m), problem.add(variable().min(0))); } } - if model.ceiling[c].failure_probability > 0.0 { - variables - .failure_probability - .insert(c, problem.add(variable().min(0))); + let ceiling = model.ceiling[c]; + let (one_value_checked, all_values_checked) = model.checked_failure_probabilities[c]; + for (failure_probability, checked, of) in [ + ( + ceiling.failure_probability_of_one_value, + one_value_checked, + &mut variables.failure_probability_of_one_value, + ), + ( + ceiling.failure_probability_of_all_values, + all_values_checked, + &mut variables.failure_probability_of_all_values, + ), + ] { + if checked && failure_probability.is_some_and(|ceiling| ceiling > 0.0) { + of.insert(c, problem.add(variable().min(0))); + } } } variables } - /// The number of $E$ and $D$ variables. + /// The number of $E$, $P$ and $W$ variables. #[cfg(test)] pub(super) fn variable_count(&self) -> usize { - self.error_bound.len() + self.failure_probability.len() + self.error_bound.len() + + self.failure_probability_of_one_value.len() + + self.failure_probability_of_all_values.len() } } /// (K) (`accuracy-models.md`, "Checking each query's target in /// selection"), per execution $c$ with rows ([`ErrorModel`]): /// - $E_{c,m} \ge \varepsilon_c[m = m_c] + \sum_i \beta_{c,i,m} -/// F_{c,i,m}$ and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per +/// F_{c,i,m}$ (one row per input instead of the sum when each output +/// value is copied from one input, `or` and a concatenation) and $F_{c,i,m} \ge E_{u,m} - M_{u,m}(1 - y_{u,c,i})$ per /// selectable edge, $F_{c,i,m} \le b_{c,i}$ where the contract's linear /// bound holds only up to $b_{c,i}$ (a ratio's denominator), and /// $E_{u,m} \le M_{u,m}(1 - y_{u,c,i})$ when $c$ has no contract in $m$ /// (the metric rule on the selected path); an input whose error does not /// reach the output ($\beta = 0$, a count's) passes none; -/// - $D_c \ge \delta_c + \sum_i H_{c,i}$, $H_{c,i} \ge D_u - M_u(1 - -/// y_{u,c,i})$ (a union bound); +/// - failure probabilities, over the inputs whose error reaches the output +/// in some metric, with $HP_{c,i} \ge P_u - M^P_u(1 - y_{u,c,i})$ and +/// $HW_{c,i} \ge W_u - M^W_u(1 - y_{u,c,i})$ (a union bound over +/// estimates, [`AccuracyRules::compose`]): +/// $P_c \ge P^{own}_c + \sum_{i\ \text{value by value}} HP_{c,i} + +/// \sum_{i\ \text{whole}} HW_{c,i}$, one row per input read value by +/// value instead of their sum when each value is copied from one input +/// (`or`), and $W_c \ge W^{own}_c + \sum_i HW_{c,i}$; /// - per query $q$ with a target, output group $g$ its root reads and /// execution $v$ answering it, $E_{v,m} \le \varepsilon_q[m \in /// \text{allowed}] + M(1 - z_{q,v})$, the metrics $g$ allows at the root /// ([`query_roots_with_target`]: the target's at an operator root, none -/// for the operands of a scalar expression no contract covers), and -/// $D_v \le \delta_q + M(1 - z_{q,v})$ with $\delta_q$ per output value -/// (`AccuracyTarget::failure_probability_per_output_value`); no $D$ row -/// without one ([`RootLimits`], each limit widened by -/// [`tolerated_limit`]). +/// for the operands of a scalar expression no contract covers); +/// - per query $q$ with a δ, one failure row by the bound +/// `AccuracyTarget::failure_probability_bound` names: $W_v \le \delta_q$ +/// (every estimate read) or $P_v \le \delta_q / V$ (each covered value) +/// $+ M(1 - z_{q,v})$; a query with several roots sums the term its +/// answering executions give per root, $T_{q,g} \ge X_v - M_v(1 - +/// z_{q,v})$, in one row $\sum_g T_{q,g} \le$ the limit. An output whose +/// term is unbounded under some selection is not answered through that +/// execution. No row without a bound ([`RootLimits`], each limit widened +/// by [`tolerated_limit`]). /// /// It is [`AccuracyRules::compose`]'s rule over variables: the same own /// error, factors, metric rule and validity caps. /// -/// $F$ and $H$ are added to `problem` here. A copy of an equivalent context +/// $F$, $HP$, $HW$ and $T$ are added to `problem` here. A copy of an equivalent context /// and its queries have no rows: they share their original's variables, /// and the copy's rows would be its original's /// ([`super::equivalent_contexts::copies_in_equivalent_contexts`] pairs @@ -636,9 +995,17 @@ pub(super) fn accuracy_constraints( let mut constraints = Vec::new(); for c in (0..model.has_rows.len()).filter(|&c| model.has_rows[c] && !model.error_is_constant[c]) { - // Σ_i β_{c,i,m} F_{c,i,m} per metric, and Σ_i H_{c,i}. - let mut error_of_inputs: [Expression; AccuracyMetric::COUNT] = Default::default(); - let mut failure_probability_of_inputs = Expression::default(); + // β_{c,i,m} F_{c,i,m} per metric and input: summed, or each alone + // when every output value is copied from one input. + let mut error_of_inputs: [Vec; AccuracyMetric::COUNT] = Default::default(); + // HP_{c,i} of each input read value by value, Σ HW_{c,i} of the + // inputs read whole, and Σ_i HW_{c,i}. + let has_one_value = accuracy.failure_probability_of_one_value.contains_key(&c); + let has_all_values = accuracy.failure_probability_of_all_values.contains_key(&c); + let how = rules.how_inputs_are_read[c]; + let mut one_value_of_inputs_read_value_by_value: Vec = Vec::new(); + let mut all_values_of_inputs_read_whole = Expression::default(); + let mut all_values_of_inputs = Expression::default(); for (i, edges) in model.selectable_edges_per_input[c].iter().enumerate() { for (m, error_of_input) in error_of_inputs.iter_mut().enumerate() { let read: Vec<(Variable, ErrorTerm)> = edges @@ -670,46 +1037,99 @@ pub(super) fn accuracy_constraints( let holds_up_to = tolerated_limit(holds_up_to); constraints.push(constraint!(of_input.clone() <= holds_up_to)); } - *error_of_input += contract.error_factor_per_input[i] * of_input; + error_of_input.push(contract.error_factor_per_input[i] * of_input); } - // H_{c,i} counts the failure of the one producer selected on - // input i, not of each readout value an output value depends - // on: `max(q)` over 1,000 series reads 1,000 readout values, - // and D here adds the readout's δ once. That is sound only - // because Pass 2's levels already split the query's δ over - // the readout values its output depends on - // (`readout_caps::query_levels`), so each readout's δ is the - // query's over 1,000; the root row checks the composed sum - // against the per-output-value δ. - let read: Vec<(Variable, ErrorTerm)> = edges - .iter() - .filter_map(|&(position, u)| { - Some((y[&position], model.failure_probability(accuracy, u)?)) - }) - .collect(); - if !read.is_empty() { - failure_probability_of_inputs += chosen_producers(&read, &mut constraints, problem); + // Only an input whose error reaches the output can make it + // miss: a count's input and the right of `and` add nothing. + if !rules + .error_metrics + .input_error_reaches_output(rules.candidate_of_execution[c], i) + { + continue; + } + let value_by_value = how.reads_value_by_value(i); + // A variable's ceiling is bounded only when every producer's + // is, so each term read here is bounded. + let read_of = |of_all_values: bool| { + edges + .iter() + .filter_map(|&(position, u)| { + let term = if of_all_values { + model.failure_probability_of_all_values(accuracy, u) + } else { + model.failure_probability_of_one_value(accuracy, u) + }; + let term = term.expect("a bounded ceiling has bounded producers")?; + Some((y[&position], term)) + }) + .collect::>() + }; + if has_one_value && value_by_value { + let read = read_of(false); + if !read.is_empty() { + one_value_of_inputs_read_value_by_value.push(chosen_producers( + &read, + &mut constraints, + problem, + )); + } + } + if has_all_values || (has_one_value && !value_by_value) { + let read = read_of(true); + if !read.is_empty() { + let of_input = chosen_producers(&read, &mut constraints, problem); + if !value_by_value { + all_values_of_inputs_read_whole += of_input.clone(); + } + all_values_of_inputs += of_input; + } } } let own = rules.own_error_per_execution[c]; for (m, of_inputs) in error_of_inputs.into_iter().enumerate() { if let Some(&error_bound) = accuracy.error_bound.get(&(c, m)) { let own = own.error_bound_per_metric[m]; - constraints.push(constraint!(error_bound >= own + of_inputs)); + if how.each_value_from_one_input && !of_inputs.is_empty() { + for of_input in of_inputs { + constraints.push(constraint!(error_bound >= own + of_input)); + } + } else { + let of_inputs: Expression = of_inputs.into_iter().sum(); + constraints.push(constraint!(error_bound >= own + of_inputs)); + } } } - if let Some(&failure_probability) = accuracy.failure_probability.get(&c) { - let own = own.failure_probability; - constraints.push(constraint!( - failure_probability >= own + failure_probability_of_inputs - )); + if let Some(&of_one_value) = accuracy.failure_probability_of_one_value.get(&c) { + let own = own + .failure_probability_of_one_value + .expect("a bounded ceiling has a bounded own term"); + let whole = all_values_of_inputs_read_whole; + if how.each_value_from_one_input && !one_value_of_inputs_read_value_by_value.is_empty() + { + for of_input in one_value_of_inputs_read_value_by_value { + constraints.push(constraint!(of_one_value >= own + of_input + whole.clone())); + } + } else { + let value_by_value: Expression = + one_value_of_inputs_read_value_by_value.into_iter().sum(); + constraints.push(constraint!(of_one_value >= own + value_by_value + whole)); + } + } + if let Some(&of_all_values) = accuracy.failure_probability_of_all_values.get(&c) { + let own = own + .failure_probability_of_all_values + .expect("a bounded ceiling has a bounded own term"); + constraints.push(constraint!(of_all_values >= own + all_values_of_inputs)); } } let memo = &inputs.physical.logical_candidates.memo; - for (&(query, group), (query_id, executions)) in memo - .output_groups() - .iter() - .zip(inputs.selectable.answering()) + let output_groups = memo.output_groups(); + // Per query with a bound and several outputs: the bound and, per + // output, what its answering execution gives. + let mut several_roots: BTreeMap = + BTreeMap::new(); + for (&(query, group), (query_id, executions)) in + output_groups.iter().zip(inputs.selectable.answering()) { debug_assert_eq!(query, query_id.0, "answering is in output_groups order"); if shared.original_of_query[query] != *query_id { @@ -718,6 +1138,8 @@ pub(super) fn accuracy_constraints( let Some(limits) = rules.root_limits(*query_id, group) else { continue; }; + let outputs_of_query = output_groups.iter().filter(|(q, _)| *q == query).count(); + let mut checked_per_execution: Vec<(Variable, ErrorTerm)> = Vec::new(); for &v in executions { let z = z[&(*query_id, v)]; for (m, &limit) in limits.tolerated_error_per_metric.iter().enumerate() { @@ -727,13 +1149,36 @@ pub(super) fn accuracy_constraints( .and_then(|error| at_most_when_answering(error, limit, z)), ); } - if let (Some(limit), Some(failure_probability)) = ( - limits.tolerated_failure_probability_per_output_value, - model.failure_probability(accuracy, v), - ) { - constraints.extend(at_most_when_answering(failure_probability, limit, z)); + let Some(bound) = limits.failure_probability_bound else { + continue; + }; + match model.failure_probability_checked_by(accuracy, v, &bound) { + Err(Unbounded) => constraints.push(constraint!(z <= 0)), + Ok(None) => {} + Ok(Some(term)) if outputs_of_query == 1 => { + constraints.extend(at_most_when_answering( + term, + tolerated_limit(bound.limit()), + z, + )); + } + Ok(Some(term)) => checked_per_execution.push((z, term)), } } + if let (Some(bound), true) = (limits.failure_probability_bound, outputs_of_query > 1) { + let of_output = if checked_per_execution.is_empty() { + Expression::default() + } else { + chosen_producers(&checked_per_execution, &mut constraints, problem) + }; + let (_, total) = several_roots + .entry(*query_id) + .or_insert_with(|| (bound, Expression::default())); + *total += of_output; + } + } + for (bound, total) in several_roots.into_values() { + constraints.push(constraint!(total <= tolerated_limit(bound.limit()))); } constraints } diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 86515e042..1f449d8bd 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -22,6 +22,7 @@ use good_lp::{ constraint, variable, Constraint, Expression, ProblemVariables, SolverModel, Variable, }; +use asap_types::accuracy::FailureProbabilityBound; use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; @@ -391,13 +392,15 @@ pub(crate) enum SelectionMilpError { Solver(String), /// The solver's selection misses `query`'s accuracy target when /// checked exactly: its limits, such as a failure probability of - /// 1e-12 per output value (δ = 0.01 over 1e10 values), are below the + /// 1e-12 for each covered value (δ = 0.01 over 1e10 values), are below the /// solver's least feasibility tolerance, /// [`LEAST_HIGHS_FEASIBILITY_TOLERANCE`], so (K) cannot be trusted. TargetBelowSolverPrecision { query: QueryId, - /// The target's δ per output value; `None` without a δ. - failure_probability_per_output_value: Option, + /// The union bound the target's δ is checked by, with its limit + /// (`AccuracyTarget::failure_probability_bound`); `None` without + /// one. + failure_probability_bound: Option, }, } @@ -556,19 +559,11 @@ pub(crate) fn solve_selection( // The query the first solve's selection missed, checked exactly: the // error names it if the second solve finds no plan at all. let mut missed_at_default_tolerance: Option = None; - let below_solver_precision = |query: QueryId| { - let memo = &inputs.physical.logical_candidates.memo; - let memo_query = &memo.queries()[query.0]; - SelectionMilpError::TargetBelowSolverPrecision { - query, - failure_probability_per_output_value: memo_query.accuracy_target.as_ref().and_then( - |target| { - target.failure_probability_per_output_value( - memo_query.number_of_values_per_query_evaluation, - ) - }, - ), - } + let below_solver_precision = |query: QueryId| SelectionMilpError::TargetBelowSolverPrecision { + query, + failure_probability_bound: rules + .as_ref() + .and_then(|rules| rules.failure_probability_bound(query)), }; let (selected, selected_edges, answering) = loop { let started = Instant::now(); diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index f446486d1..aca48c406 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeSet, HashMap}; use asap_cost::node::analytical::AnalyticalCostError; use asap_cost::{summary_type_name, CostModel}; -use asap_types::accuracy::AccuracyMetric; +use asap_types::accuracy::{AccuracyMetric, FailureProbabilityBound}; use asap_types::deployment_capabilities::ExclusionSource; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator}; @@ -223,14 +223,14 @@ pub enum ErrorSelectingPlan { /// Query `query`'s accuracy target is below the solver's precision: /// the MILP's selection misses it when checked exactly, so no plan is /// returned rather than one that may not meet the target. Example: a - /// failure probability of 1e-12 per output value (δ = 0.01 over 1e10 - /// values), below the solver's feasibility tolerance 1e-10. + /// failure probability of 1e-12 for each covered value (δ = 0.01 over + /// 1e10 values), below the solver's feasibility tolerance 1e-10. #[error( - "query {query}: an accuracy target is below the solver's precision (failure probability per output value {failure_probability_per_output_value:?}); the solver's plan misses it, so no plan is returned" + "query {query}: an accuracy target is below the solver's precision (failure probability bound {failure_probability_bound:?}); the solver's plan misses it, so no plan is returned" )] AccuracyTargetBelowSolverPrecision { query: usize, - failure_probability_per_output_value: Option, + failure_probability_bound: Option, }, /// The selection MILP implements cost model 1 only (#1095 §3). #[error("the selection MILP does not support {0}")] diff --git a/crates/types/src/accuracy/mod.rs b/crates/types/src/accuracy/mod.rs index 67546063e..c0ae14bec 100644 --- a/crates/types/src/accuracy/mod.rs +++ b/crates/types/src/accuracy/mod.rs @@ -11,5 +11,6 @@ mod target; pub use default_metric::default_metric_for_root; pub use guarantee::{AccuracyGuarantee, BoundExpr, ProbabilityExpr}; pub use target::{ - AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, ReadoutValueCounts, + AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, FailureProbabilityBound, + NormScale, ReadoutValueCounts, }; diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs index 42efa60f2..b16c8ab25 100644 --- a/crates/types/src/accuracy/target/mod.rs +++ b/crates/types/src/accuracy/target/mod.rs @@ -167,9 +167,9 @@ impl AccuracyScope { /// when unbound: one for [`Self::PerValue`], `G` for /// [`Self::WholeResult`], and `min(⌊1 / share⌋, G)` for /// [`Self::GroupsWithRecordShareAtLeast`], which is bound even when `G` - /// is not. A target's failure probability splits over the readout values + /// is not. A target's failure probability is bounded over the estimates /// these values depend on - /// ([`AccuracyTarget::failure_probability_per_readout_value`]), so the + /// ([`AccuracyTarget::failure_probability_bound`]), so the /// strictest of several scopes is the one that covers the most values. pub fn max_values_covered(self, values_per_result: Option) -> Option { match self { @@ -298,30 +298,6 @@ impl AccuracyTarget { self.max_error == 0.0 && self.failure_probability == Some(0.0) } - /// The failure probability each output value of one evaluation must - /// meet: the target's δ split evenly, by the union bound, over the - /// values its scope covers when one evaluation returns - /// `values_per_evaluation` ($n_q$; [`AccuracyScope::max_values_covered`]). - /// `None` when the target sets no δ, and when that count is not known - /// for a δ over the whole result. A readout value's failure probability - /// splits δ over the readout values the covered output values depend - /// on instead ([`Self::failure_probability_per_readout_value`]). - /// - /// Example: δ = 0.01 over the whole result of a query returning 100 - /// series gives 0.0001 per output value; with the number of series - /// unknown, `None`; per value (`AccuracyScope::PerValue`), 0.01 either - /// way. - pub fn failure_probability_per_output_value( - &self, - values_per_evaluation: Option, - ) -> Option { - let delta = self.failure_probability?; - let values = self - .accuracy_scope - .max_values_covered(values_per_evaluation)?; - Some(delta / values.max(1) as f64) - } - /// The metrics this target accepts an error in: its metric, every /// metric when it names none. pub fn metrics(&self) -> Vec { @@ -331,17 +307,62 @@ impl AccuracyTarget { .collect() } - /// The failure probability each readout value below the query must - /// meet: δ split by the union bound over the readout values the output - /// values one statement covers depend on, `readout_values`, when one - /// evaluation returns `values_per_evaluation` ($n_q$). 1 when the - /// target sets no δ; `None` when no bound on that count is known. + /// Which union bound checks this target's failure probability, and + /// its limit, when one evaluation returns `values_per_evaluation` + /// ($n_q$) and the query reads `readout_values` (`R` and the fan-in). + /// `None` when the target sets no δ, and when neither bound is known. /// - /// The statement covers at most `V` output values - /// ([`AccuracyScope::max_values_covered`]), each depending on at most - /// `fan_in` readout values, and the query reads `R` in all, so it fails - /// only when one of at most `min(R, V · fan_in)` readout values does, - /// whatever their dependence. An unknown term drops out of the `min`. + /// The statement fails only when an estimate (one readout value in one + /// evaluation) a covered output value depends on fails, so its failure + /// probability is at most the summed δ of those estimates. Two sums + /// bound it: every estimate the query reads (at most `R`), or each of + /// the `V` covered values' own estimates (at most `V · fan-in`; + /// [`AccuracyScope::max_values_covered`]). The bound counting fewer + /// estimates is named; a tie, or `V · fan-in` unknown, names every + /// estimate read; `R` unknown names each covered value. + /// + /// Examples at δ = 0.01 over the whole result: + /// - `max(quantile_over_time(0.9, m[5m]))` over 1000 series: `R` 1000, + /// `V · fan-in` 1 · 1000, so every estimate read, at most 0.01. + /// - `q or r` over 100 series each, same labels: `R` 200, `V · fan-in` + /// 100 · 1, so each covered value, at most 0.01 / 100. + /// - `q * on() group_left e`, `e` one sketched series, 1000 values: + /// `R` 1, so every estimate read, at most 0.01. + pub fn failure_probability_bound( + &self, + values_per_evaluation: Option, + readout_values: ReadoutValueCounts, + ) -> Option { + let delta = self.failure_probability?; + let values_covered = self + .accuracy_scope + .max_values_covered(values_per_evaluation); + let estimates_of_covered_values = values_covered + .zip(readout_values.per_output_value) + .map(|(values, fan_in)| values.saturating_mul(fan_in)); + let every_estimate_read = FailureProbabilityBound::EveryEstimateRead { limit: delta }; + Some( + match (readout_values.per_evaluation, estimates_of_covered_values) { + (Some(read), Some(covered)) if read <= covered => every_estimate_read, + (Some(_), None) => every_estimate_read, + (_, Some(_)) => { + let values_covered = values_covered?.max(1); + FailureProbabilityBound::EachCoveredValue { + values_covered, + limit: delta / values_covered as f64, + } + } + (None, None) => return None, + }, + ) + } + + /// The failure probability each readout value below the query must + /// meet: the limit of [`Self::failure_probability_bound`] split evenly + /// over the estimates its sum counts, `R` for every estimate read and + /// the fan-in for each covered value. So a query whose readouts all + /// meet it meets its bound by construction. 1 when the target sets no + /// δ; `None` when no bound is known. /// /// Examples at δ = 0.01: /// - `quantile_over_time(0.99, m[5m])` over 8 series per value: @@ -355,19 +376,44 @@ impl AccuracyTarget { values_per_evaluation: Option, readout_values: ReadoutValueCounts, ) -> Option { - let Some(delta) = self.failure_probability else { + if self.failure_probability.is_none() { return Some(1.0); - }; - let readout_values_covered = self - .accuracy_scope - .max_values_covered(values_per_evaluation) - .zip(readout_values.per_output_value) - .map(|(values, fan_in)| values.saturating_mul(fan_in)); - let split_over = match (readout_values.per_evaluation, readout_values_covered) { - (Some(all), Some(covered)) => all.min(covered), - (all, covered) => all.or(covered)?, - }; - Some(delta / split_over.max(1) as f64) + } + Some( + match self.failure_probability_bound(values_per_evaluation, readout_values)? { + FailureProbabilityBound::EveryEstimateRead { limit } => { + limit / readout_values.per_evaluation?.max(1) as f64 + } + FailureProbabilityBound::EachCoveredValue { limit, .. } => { + limit / readout_values.per_output_value?.max(1) as f64 + } + }, + ) + } +} + +/// Which union bound a query's failure probability is checked by, with its +/// limit ([`AccuracyTarget::failure_probability_bound`]). Pass 2 splits δ +/// by it, and selection checks it on the readouts actually selected. +/// +/// Example: δ = 0.01 over the whole result of `q or r`, 100 series each: +/// `EachCoveredValue { values_covered: 100, limit: 1e-4 }`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum FailureProbabilityBound { + /// The summed δ of every estimate any output value depends on is at + /// most `limit`, the target's δ. + EveryEstimateRead { limit: f64 }, + /// The summed δ of the estimates one output value depends on is at + /// most `limit`, δ over the `values_covered` values the scope covers. + EachCoveredValue { values_covered: u64, limit: f64 }, +} + +impl FailureProbabilityBound { + /// Its limit. + pub fn limit(&self) -> f64 { + match *self { + Self::EveryEstimateRead { limit } | Self::EachCoveredValue { limit, .. } => limit, + } } } diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 2a108ebca..7538c9a15 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -178,6 +178,35 @@ impl Memo { smaller_known(by_keys, by_rows) } + /// The most estimates (sketch readout values) one evaluation of the + /// readout group `group` computes: its values + /// ([`Self::max_values_per_evaluation`]), except for a top-k, which + /// estimates every distinct item of each state it ranks. That is at + /// most its states' count times the distinct items one state counts + /// ([`Self::max_distinct_per_evaluation`]), and at most the rows (or + /// series) its values are bounded by, as each item comes from a row; + /// the smaller known one. This is the `R` a readout's failure + /// probability is charged for when an output reads all its values. + /// + /// Example: `topk(10, sum by (dst) (count_over_time(flows[5m])))` as + /// one heap per `job`, with 3 jobs and at most 500 distinct `dst` per + /// evaluation declared, estimates at most 3 × 500 = 1500 items, while + /// it returns 30 values. + pub fn max_estimates_per_evaluation(&self, group: MemoGroupId) -> Option { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query: SketchStatistic::TopK { .. }, + }) = &self.groups[group.0 as usize].expression + else { + return self.max_values_per_evaluation(group); + }; + let by_distinct_items = self + .max_values_per_evaluation(*summary_input) + .zip(self.max_distinct_per_evaluation(group)) + .and_then(|(states, items)| states.checked_mul(items)); + smaller_known(by_distinct_items, self.max_values_per_evaluation(group)) + } + /// The most rows one evaluation of `group` returns, from the facts the /// data workload declares; `None` when they bound it in no way. A `Scan` /// returns the rows its source holds (`SourceFacts::rows_held`), and a diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index b4a8bc108..0936b48fe 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -79,7 +79,9 @@ heap size and its value guarantee alone (the contracts table). Pass 2 gives each sketch readout the error and per-value failure probability `δ / min(R, V · fan_in)` it must meet -(`AccuracyTarget::failure_probability_per_readout_value`): the statement +(`AccuracyTarget::failure_probability_per_readout_value`, the limit of the +bound `AccuracyTarget::failure_probability_bound` names split over the +estimates that bound counts): the statement fails only when a readout value one of its `V` covered output values depends on does. `R` is the readout values the query reads per evaluation (the values per evaluation of every readout group below it, summed), and `fan_in` @@ -89,7 +91,11 @@ for a global `max`; for `sum by (job)` too, since the declared facts bound how many jobs there are, not how many series one job holds), the sum over the sides of a binary operator, the larger side of `or` or input of a concatenation. A top-k readout chooses its output keys by their estimates, -so each of its output values depends on every value it ranks. A readout +so each of its output values depends on every item it ranks, and its `R` +counts those items, not its output values +(`Memo::max_estimates_per_evaluation`): at most its states' count times the +distinct items one state counts (`SourceFacts::max_distinct_per_evaluation`), +and at most the rows its values come from. A readout group whose candidates include a reduction over readouts below it takes the larger of the two counts. Both counts are per evaluation, not per subquery step. An unknown term drops out of the `min`; with both unknown no per-value @@ -106,7 +112,14 @@ one: DDSketch's relative error does not depend on δ, so of (5%, 0.01) and build per listed θ, and each edge checks the guarantee of its producer's θ at the required failure probability, so a build several readers share serves the readers whose levels its θ meets. Stage 3 only checks the guarantee; it -never resizes. At equal δ, the scope covering the most values is the +never resizes. Stage 3's constraint (K) bounds the same estimates with the +same counts (`R` and the fan-in, from one walk, +`asap_accuracy::readout_value_counts_of_queries`, and one test of whether an +input's error reaches the output), so for each query planned alone a +readout at this split meets (K). A level another query gave is checked, not +trusted ("Which estimates a result depends on", below). When two queries +share an execution and need incompatible levels, no level is given for the +pair, and the selection may find no sketch plan for one of them. At equal δ, the scope covering the most values is the strictest: every group (`G`, when `G` is above 100), then a 1% share (100), then a 5% share (20), then a single value (1). A whole-result target with an unbound `G` splits over `R`, so `R` must be an upper bound. A readout @@ -300,17 +313,24 @@ has no value bound, so neither passes through division. Likewise, an exact sum above approximate values retains their uncertainty: exact arithmetic does not recover information lost below it. -Probability is not propagated through operators. A query's failure -probability is split into a per-value one for each readout; a composed query -whose readouts' failure events add up (a ratio of two quantiles) reads each at -a level of a failure-probability grid, and the levels sum within the query's -per-value failure probability by the union bound, without assuming -independence between child errors. Estimator-local assumptions, such as -independent bucket hashing, must be stated separately. Shared DAG nodes do -not create independent errors merely because several consumers reference -them, and sharing does not justify reducing the failure budget. Per-result -guarantees also do not imply a simultaneous guarantee across every query or -evaluation time in a dashboard. +Failure probability is not a contract factor: an operator does not scale +it. What fails is an estimate, one value a sketch readout returns in one +evaluation (one series' quantile, one group's distinct count, one key's +frequency), which misses its error bound with probability at most its +readout's δ. A query's result fails only when an estimate it depends on +does, so its failure probability is at most the sum of δ over the estimates +it can depend on (the union bound, without assuming independence between +estimates). Pass 2 splits a query's δ over those estimates; selection checks +the sum for the readouts actually selected ("Which estimates a result +depends on"). A composed query (a ratio of two quantiles) reads each readout +at a level of a failure-probability grid and the selection combines them. +Estimator-local assumptions, such as independent bucket hashing, must be +stated separately. Shared DAG nodes do not create independent errors merely +because several consumers reference them. Sharing never earns a larger δ for +an estimate: an estimate several output values read is one event, which is +why it is counted once per path, not once per reader. Per-result guarantees +also do not imply a simultaneous guarantee across every query or evaluation +time in a dashboard. ### Sign conditions @@ -425,9 +445,24 @@ each query's target. This is constraint (K). **Variables.** Per execution `c` (a candidate in one evaluation context) that a query with a target reaches, and that is its own original (not in a -copy of an equivalent context): its error bound `E_{c,m}` per metric `m` and -its failure probability `D_c`, continuous; per input `i`, `F_{c,i,m}` and -`H_{c,i}`, what it reads from the producer selected on `i`. +copy of an equivalent context), all continuous: +- its error bound `E_{c,m}` per metric `m`; +- `P_c`, the failure probability of one of its output values: the largest, + over its output values, of the summed δ of the estimates that value + depends on; +- `W_c`, the failure probability of its whole output: the summed δ of every + estimate any of its output values depends on. It exists only where every + readout below `c` bounds its estimates per evaluation (`R` known). + +Each exists only where some selection makes it positive and its error is +not constant. Per input `i`, `F_{c,i,m}`, `HP_{c,i}` and `HW_{c,i}` are what +`c` reads from the producer selected on `i`. `P` and `W` exist only where a +query root above checks them, through the rows that read them: a root +checking each covered value reads `P`, one checking every estimate read +`W`; `P` reads `P` of an input read value by value and `W` of one read +whole; `W` reads `W` (`ErrorModel::checked_failure_probabilities`). With +both created wherever bounded, RangeHeavy-8users solved about 3× slower +with the same plan. **Rows.** With `y_{u,c,i}` the edge variables and `z_{q,v}` the choice of execution `v` answering query `q`: @@ -436,45 +471,183 @@ execution `v` answering query `q`: `F_{c,i,m} ≥ E_{u,m} − M_{u,m}(1 − y_{u,c,i})` per selectable producer `u`; `ε_c` is a readout's own error, its required accuracy (`CandidateForMemoGroup::required_accuracy`), 0 for any other candidate. + An operator that copies each output value from one input (`or`, a + concatenation) has one row per input instead of the sum: its output's + error is one input's, `E_{c,m} ≥ ε_c[m = m_c] + β_{c,i,m} F_{c,i,m}` for + each `i`. - The metric rule on the selected path: `E_{u,m} ≤ M_{u,m}(1 − y_{u,c,i})` when `c` has no contract in `m`. An input with `β = 0` passes none. - The validity cap: `F_{c,i,m} ≤ b_{c,i}` where the contract's linear bound holds only up to `b` (a ratio's denominator). -- `D_c ≥ δ_c + Σ_i H_{c,i}`, `H_{c,i} ≥ D_u − M_u(1 − y_{u,c,i})`: a union - bound over the producers selected. -- At each output of a query with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` +- Failure probabilities, with `HP_{c,i} ≥ P_u − M^P_u(1 − y_{u,c,i})` and + `HW_{c,i} ≥ W_u − M^W_u(1 − y_{u,c,i})` per selectable producer `u`, over + only the inputs whose error reaches the output (`β > 0` in a metric of + `c`'s contract, the test Pass 2's split uses; a count's input and the + right input of `and` and `unless` add nothing): + - a readout: `P_c = δ_c` and `W_c = R_c δ_c`, with `δ_c` its required + failure probability and `R_c` its estimates per evaluation + (`Memo::max_estimates_per_evaluation`); a top-k readout, whose output + keys are chosen by every estimate it ranks, has `P_c = W_c = R_c δ_c`. + Constants, as a readout's error is; + - any other execution: + `P_c ≥ Σ_{i read value by value} HP_{c,i} + Σ_{i read whole} HW_{c,i}`, + where an operator that copies each output value from one input (`or`, a + concatenation) has instead `P_c ≥ HP_{c,i} + Σ_{i read whole} HW_{c,i}` + for each input `i` read value by value; and `W_c ≥ Σ_i HW_{c,i}`. The + operator classes are in the table below. +- At the outputs of a query `q` with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` in each metric the root allows (the target's at an operator root, none for - the operands of a scalar expression no contract covers, 0 elsewhere), and - `D_v ≤ δ_q / V + M(1 − z_{q,v})` with `V` the values the target's scope - covers (`AccuracyTarget::failure_probability_per_output_value`, `n_q` for a - whole result); no `D` row without a δ. Each limit is widened as the edge - check widens it (`tolerated_limit`). - -`D` adds each selected producer's δ once, not once per readout value an -output value depends on: `max(q)` over 1,000 series reads 1,000 readout -values. That is sound because Pass 2's levels already split the query's δ -over the readout values its output can depend on, so each readout's δ is the -query's over 1,000. - -**Constant errors.** A readout's state is exact, so its error is its own -requirement under every selection: it has no `E` or `D` variable, and its -consumers read `Σ_u ε_u y_{u,c,i}` (at most one `y` of an input is 1). This is + the operands of a scalar expression no contract covers, 0 elsewhere). With + a δ, one failure row per query, over every output `v` it may be answered + from, by the bound `AccuracyTarget::failure_probability_bound` names from + `V` (the values the target's scope covers) and the query's counts `R` and + fan-in (Pass 2's `ReadoutValueCounts`): + - every estimate read, when `R ≤ V · fan_in` or `V` is unbound: + `W_v ≤ δ_q + M(1 − z_{q,v})`; + - each covered value, when `V · fan_in < R` or `R` is unbound: + `P_v ≤ δ_q / V + M(1 − z_{q,v})`; + - neither, when both are unbound: no row. Pass 2 gave the query no level, + and constraint (X) makes it read the readout groups below it only + exactly; that, not (K), is what keeps it from reading a level another + query gave. + + An output whose checked term is unbounded under some selection (a readout + below with unknown `R`) is not answered by that execution. A query with + several roots (a scalar expression over them) sums its roots' terms in its + one row. Each limit is widened as the edge check widens it + (`tolerated_limit`). + +**Which estimates a result depends on.** The union bound is taken over +estimates, not over readouts or output values. + +*Soundness.* Every operator above a readout is deterministic, and its error +contract holds whenever its inputs are within their bounds. So an output +value misses `ε_q` only if an estimate it depends on misses its bound. Which +estimates an output value depends on is decided by the data's series and +keys (labels, group keys, join keys), not by the estimates. The exceptions +are operators that choose by estimated value: a top-k readout is charged for +every value it ranks, and any other such operator (a comparison, a `LIMIT` +after ordering by an estimate) has no contract. So for any data `D` the +statement fails only on `⋃_{e ∈ S_q(D)} E_e`, where `S_q(D)` is the +estimates the covered output values depend on and `E_e` is estimate `e`'s +miss. Hence + + P(fail) ≤ max_D Σ_{e ∈ S_q(D)} δ_e. + +Both root rows bound the right side for every `D` the declared facts allow: +- every estimate read: `S_q(D)` lies in the estimates of the readouts below, + at most `R_c` per readout per path, so the sum is at most `W_v`; +- each covered value: `S_q(D) = ⋃_{v covered} deps(v)`, at most `V` values, + each with `Σ_{e ∈ deps(v)} δ_e ≤ P_v`, so the sum is at most `V · P_v`. + +The recursions are sound step by step: +- a value read value by value is one input value, whose estimates sum to at + most `P_u`; +- a value copied from one input (`or`) is that input's value, so the largest + of the inputs' `P` bounds it; +- an input read whole (a reduction) may contribute every estimate of that + input, at most `W_u`; +- a whole output depends on at most the union of its inputs' estimates, at + most `Σ W_u`. + +An exact input has δ = 0. An input with `β = 0` contributes only which series +or keys it has. A readout returns exactly the series the exact evaluation +does, so those estimates cannot make the output miss. + +*The assumption it rests on.* Soundness needs `R_c` to be an upper bound on +the estimates one evaluation of readout `c` computes, under every dataset +the declared facts allow; it is the only declared count the bound trusts +(`n_q` only decides which row is checked). `R_c` is +`Memo::max_values_per_evaluation`: the product of the distinct values its +state's keys' scans declare, at most the series for PromQL labels, clamped +to the rows its input reads. Two cases checked against it: +- A top-k readout ranks every distinct item of each state, more than the + values it returns. Its `R_c` is the smaller of its states' count times the + declared distinct items per state (`SourceFacts::max_distinct_per_evaluation`) + and the series or rows its items come from, both upper bounds on the items + (`Memo::max_estimates_per_evaluation`). +- A time-bucketed SQL grouping (`GROUP BY src, date_trunc('minute', ts)`) + returns buckets × keys values. A time or computed key bounds nothing by + itself, so `R_c` falls back to the rows the grouping reads, which bounds + the groups whatever the buckets. The executed bounds check + (`memo_value_bounds`) runs these shapes on data the facts allow. + +*Scopes.* `PerValue` covers one value: `V = 1`, so the row is `P_v ≤ δ`, with +no division by `n_q`. `WholeResult` covers `V = n_q`. +`GroupsWithRecordShareAtLeast(s)` covers `V = min(⌊1/s⌋, G)` groups. Which +groups those are is decided by exact record counts, not by estimates, so the +same two rows hold with that `V`, and `W_v` (all groups) stays a sound +alternative. + +*Operator classes* (shared with Pass 2's split: +`asap_accuracy::inputs_read_value_by_value`, +`takes_each_value_from_one_input` and the "error reaches the output" test +`ErrorMetricsOfMemo::input_error_reaches_output`): + +| Class | Operators | `P_c` | `W_c` | +|---|---|---|---| +| Readout | `SummaryEstimate` (not top-k) | `δ_c` | `R_c δ_c` | +| Readout choosing by estimate | top-k | `R_c δ_c` | `R_c δ_c` | +| Value by value | filter on copied columns, project, sort, time shift, relabel, label enrichment, series sample; binary arithmetic, one-to-one or many-to-one (`on`, `ignoring`, `group_left`, `group_right`: each output value reads one value of each side, and the "one" side's value is read by many outputs but is one estimate); join | `Σ_i HP` | `Σ_i HW` | +| Each value from one input | `or`, concatenation (`UNION ALL`); `and`, `unless` (left only, the right has `β = 0`) | `max_i HP` | `Σ_i HW` | +| Input read whole | aggregates (`sum`/`max`/`avg` by or without keys), any operator whose output value may read many values of an input, the operand of `scalar` | `Σ_i HW` | `Σ_i HW` | +| No contract | a subquery, a comparison, a counter rate | none: no error, and so no estimate, passes | none | +| `β = 0` input | a count's input, the right of `and` and `unless` | adds 0 | adds 0 | + +*Same rule as Pass 2, for a query planned alone.* Pass 2 gives each readout +of a query one level `δ / min(R, V · fan_in)`. Its `R` and fan-in add, per +path, the terms `W` and `P` add, and take the largest over a group's +candidates where the rows take the selected one. So at that level, +`W_v ≤ R · δ / min(…)` and `V · P_v ≤ V · fan_in · δ / min(…)`. The row the +root checks is the one whose count is the `min`, so it is at most δ. A level +a stricter rule gave (any `δ_c ≤ δ / max(V, R)`) meets it too. A level given +for another query with a looser split does not: `max(q)` over 1,000 series +cannot read a copy at the 0.01 a per-value query on `q` was given, because +`W = 1,000 × 0.01 > 0.01`. + +*What the rows lose* (all sound; each only asks for a stricter δ than +needed): +1. An estimate one query reaches on several paths counts once per path: + `max(q) − min(q)` over 1,000 series charges `2,000 δ_c`, although 1,000 + estimates fail at most. Pass 2's `R` counts it the same way, so no level + it gives becomes infeasible. +2. The root checks one of the two rows, chosen from the counts. When the + readouts below have different δs (a composed query), the other row may + have been smaller. +3. An operator that returns fewer values than it reads does not lower the + count of a whole input (a label filter above a shared readout, a join + dropping unmatched rows), as in Pass 2's `R`. + +Counting each estimate exactly once, per query, needs a variable per +(query, execution) on the query's selected path, and at each `or` a dual +variable bounding how many output values each side supplies. That is linear +but grows with queries × executions; it is not used. + +**Constant errors.** A readout's state is exact, so its error and failure +probabilities are its own under every selection: it has no `E`, `P` or `W` +variable, and its consumers read `Σ_u ε_u y_{u,c,i}`, `Σ_u δ_u y_{u,c,i}` and +`Σ_u R_u δ_u y_{u,c,i}` (at most one `y` of an input is 1). This is exactly the variable form with `E_u = ε_u`, and it removes the big-M rows of every readout. -**Ceilings.** `M` is the largest error any selection gives an execution, +**Ceilings.** `M` (and `M^P`, `M^W`) is the largest error and failure +probability any selection gives an execution (unbounded for `P` or `W` when +a readout below has unknown `R`; then the execution has no such variable), computed in one pass in the memo's topological order (every selectable edge reads a group ordered before the consumer's), each input taking its largest producer's ceiling, by the same composition rule as the rows. The rule is -written once (`AccuracyRules::compose`), and also checks a whole selection -(`AccuracyRules::selection_meets_targets`): the greedy upper bound that prunes +written once (`AccuracyRules::compose`, composing `E`, `P` and `W`), and +also checks a whole selection (`AccuracyRules::selection_meets_targets`, the +same root row): the greedy upper bound that prunes executions dearer than a feasible plan uses its plan only when it meets (K), since the cheapest readouts of a ratio may each meet their level and compose above the target. **Solver tolerance.** HiGHS's feasibility tolerances are absolute, 1e-7 by -default, while a per-value δ is often far smaller: δ = 0.01 over a whole -result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth: two +default. A row checking every estimate read compares sums of the size of δ +itself (0.01 against 0.01), where 1e-7 is negligible. A row checking each +covered value compares `P_v` with δ / V, often far smaller: δ = 0.01 over a +whole result of 1e4 values is 1e-6 per value, of which 1e-7 is a tenth: two readouts failing with 5.4e-7 each compose to 1.08e-6 and pass that limit within the default tolerance. So HiGHS first solves at its defaults, and the selection is then checked exactly (`AccuracyRules::selection_meets_targets`, @@ -484,8 +657,8 @@ A row may still pass its limit by up to 1e-10, which for a per-value δ near or below 1e-10 can be orders of magnitude: at δ = 1e-12 per value (δ = 0.01 over 1e10 values) the solver may select readouts failing with 1.08e-12 in all. A selection that misses a target after the second solve is an error -naming the query and its per-value δ ("an accuracy target is below the -solver's precision"); planning never returns a plan that may not meet its +naming the query and its bound with its limit ("an accuracy target is below +the solver's precision"); planning never returns a plan that may not meet its target. The stricter tolerance is paid for only when the check needs it: solving every (K) at 1e-10 made RangeHeavy-8users about 6× slower, deriving the tolerance from the smallest limit solved RollupHeavy-8users about 2.8× @@ -494,10 +667,14 @@ about 4× slower. **Context copies.** A copy of an equivalent evaluation context shares its original's variables and has no rows of its own. Two contexts are copies only -when each output's root limits (the tolerated error per metric and failure -probability per output value) are equal, so the copy's rows would repeat the -original's: queries at δ = 0.01 per value returning 10 and 100 values are -copies; over a whole result they are not. +when each output's root limits (the tolerated error per metric, and the +failure probability bound with its limit and, for each covered value, its +`V`) are equal, so the copy's rows would repeat the original's: +- queries at δ = 0.01 per value returning 10 and 100 values are copies; +- over a whole result they are copies when both check every estimate read + (a readout of 10 series declaring 10 and 50 values); +- they are not copies when they check each covered value of 10 and of 100 + values. ### Example: allocation in a ratio @@ -517,6 +694,30 @@ levels of the grid up to its cap; with levels 0.02 (cost 1) and 0.01 (cost 3): (K) selects both 0.01 readouts, cost 6. A greedy plan reading the two cheapest readouts (cost 2) misses B's target, so it bounds nothing. +### Example: failure probability in `or`, many-to-one matches and reductions + +All at δ = 0.01 over the whole result unless stated. + +| Query | Facts | Pass 2's level (`δ / min(R, V · fan_in)`) | Row (K) checks | Value | Estimates that can fail at once | +|---|---|---|---|---|---| +| `q or r` | 100 series each, same labels, `n_q` = 100 | `R` = 200, fan-in 1: `0.01 / min(200, 100) = 1e-4` | each covered value: `max(1e-4, 1e-4) ≤ 0.01 / 100` | 1e-4 ✓ | 100 (each output copies one) | +| `q * on() group_left e` | `q` exact over 1,000 series, `e` one sketched series, `n_q` = 1,000 | `R` = 1, fan-in 1: `0.01 / min(1, 1,000) = 0.01` | every estimate read: `W = 0 + 1 · 0.01` | 0.01 ✓ | 1 (every output reads `e`'s one estimate) | +| `max(q)` | 1,000 series, `n_q` = 1 | `R` = 1,000, fan-in 1,000: `1e-5` | every estimate read: `1,000 · 1e-5` | 0.01 ✓ | 1,000 | +| `count(distinct user_id)` by service, per value | δ = 0.05, ε = 0.05 relative, 10,000 services | `R` = 10,000, `V` = 1, fan-in 1: `0.05 / 1 = 0.05` | each covered value, `V` = 1: `P = 0.05` | 0.05 ✓ | 1 per value | +| `A or B`, `A` = `quantile_over_time(0.99, m{env="prod"}[5m])`, `B` = `quantile_over_time(0.99, m[5m])`, both reading one readout of `m` | 100 series, `n_q` = 100 | `R` = 200 (one readout, two paths), fan-in 1: `1e-4` | each covered value: `max(1e-4, 1e-4) ≤ 0.01 / 100` | 1e-4 ✓ | 100 | +| `max by (job) (A or B)` | as above, 10 jobs | `R` = 200, fan-in 200: `5e-5` | every estimate read: `W = 200 · 5e-5` | 0.01 ✓ | 100: the rows charge each estimate twice (once per path), so 1e-4 would have sufficed | + +Charging the selected readout's δ once against δ / `n_q` would instead: +- reject `q or r` (`2e-4 > 1e-4`) and the many-to-one match + (`0.01 > 0.01 / 1,000`), leaving them no sketch plan; +- accept for `max(q)` any copy of the readout, including one a per-value + query on `q` was given at 0.01, whose 1,000 estimates may fail with up to + 1,000 × 0.01. + +For the distinct count, a split over `max(V, R)` would ask each group's KMV +for 0.05 / 10,000 = 5e-6, k = 26,656. The per-value row asks only that each +group's own estimate fail with at most 0.05, which KMV meets at k = 4,323. + ## Evidence and trust boundaries `AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the From 5e93d34c835058fa21201ea0df91e79ad7d25595 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:42:17 +0000 Subject: [PATCH 4/5] test(accuracy): constraint (K) charges a set operator for the top-k choosing its values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `p90 and topk(3, …)`, `p90 unless topk(3, …)` and `topk(3, …) or p90` keep or take a value by which keys the top-k returns, which all of its ranked estimates decide: each output value's failure probability adds the top-k's to the p90's. Fails today: (K) drops the right side of and/unless and takes the larger side of or. --- .../src/milp/model/tests.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 365d98210..d67dce47c 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -2005,6 +2005,76 @@ fn k_charges_a_top_k_readout_for_every_estimate_it_ranks() { } } +/// The top-k readout of 10 series of `m` (CMS with heap, k = 3) failing +/// with `top_k_level` per estimate, and the p90 readout of 10 series of +/// `r` failing with `p90_level`, combined by the set operator `kind` with +/// the p90 readout on the left, read per value at δ = 0.01; `or` with the +/// top-k on the left. `None` when no plan meets the target. +fn set_operator_over_top_k( + kind: PromQLVectorSetOpKind, + top_k_level: f64, + p90_level: f64, +) -> Option> { + let mut memo = memo_with_series(&[("m", 10), ("r", 10)]); + let (top_k, _) = readouts_of( + &mut memo, + "m", + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 3 }, + AccuracyMetric::Relative, + &[(0.01, top_k_level)], + ); + let (p90, _) = readouts_of( + &mut memo, + "r", + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.9 }, + AccuracyMetric::Relative, + &[(0.01, p90_level)], + ); + let (left, right) = if matches!(kind, PromQLVectorSetOpKind::Or) { + (top_k, p90) + } else { + (p90, top_k) + }; + let set = group(&mut memo, memo_fixtures::set_operator(kind, left, right)); + add_query( + &mut memo, + set, + relative(0.05) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue), + Some(10), + ); + selected_or_infeasible(&memo) +} + +/// `p90 and topk(3, …)` and `p90 unless topk(3, …)` keep a p90 value +/// only where the top-k's output has (or lacks) its labels, and which +/// keys the top-k returns is chosen by all 10 estimates it ranks: each +/// output value fails when its p90 estimate or any of those does. The +/// top-k at 1e-3 (10 · 1e-3 = 0.01) with the p90 at 0.005 sums to 0.015, +/// above δ = 0.01; at 1e-4 (0.001) the sum is 0.006, within it. +/// `topk(3, …) or p90` takes a p90 value only where the top-k returns no +/// such key, so each value depends on both sides' estimates: the same +/// sums, not the larger side. +#[test] +fn k_charges_a_set_operator_for_the_top_k_choosing_its_values() { + for kind in [ + PromQLVectorSetOpKind::And, + PromQLVectorSetOpKind::Unless, + PromQLVectorSetOpKind::Or, + ] { + for (top_k_level, plans) in [(1e-3, false), (1e-4, true)] { + assert_eq!( + set_operator_over_top_k(kind.clone(), top_k_level, 0.005).is_some(), + plans, + "{kind:?} {top_k_level}" + ); + } + } +} + /// The p90 and the p50 readouts of 100 series concatenated, per value at /// δ = 0.01: each row comes from one readout, so it fails only with that /// readout's estimate: readouts at 0.01 meet it, at 0.02 they do not. From bf6288565203f06d2366f0fe29aa1ab2788a9b7e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:57:05 +0000 Subject: [PATCH 5/5] fix(accuracy): (K) reads Pass 2's counts and counts the top-k choosing a set operator's values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass 2 records the readout values (R, fan-in) each query's δ is split over (LogicalASAPMemoCandidates::readout_values_of_query, from ReadoutLevels); Stage 3 checks the bound they name (LogicalASAPMemoCandidates::failure_probability_bound) instead of walking the memo again. readout_value_counts_of_queries is deleted. - An input whose values a top-k chooses (values_chosen_by_estimate) counts its estimates even where beta = 0 (the right of and/unless), and `or` sums its sides' one-value failure probabilities when one side's values are chosen by estimate. One rule for Pass 2's count walk and (K): ErrorMetricsOfMemo::input_estimates_reach_output and each_value_depends_on_one_input. - A readout required at delta = 0 adds 0 whatever its estimate count. - RootLimits compares the bound's kind and limit only (values_covered is not read by any row); the precision error prints the bound in words (FailureProbabilityBound: Display). - Docs: the count walk, R = Memo::max_estimates_per_evaluation and its distinct-item contract, context copies, and the measured cost of scaling failure probabilities by the tightest limit (not adopted). - accuracy_constraints counts each query's outputs once. --- crates/accuracy/src/estimate_dependence.rs | 41 ++++- crates/accuracy/src/lib.rs | 2 +- crates/accuracy/src/metric_rule/mod.rs | 52 ++++-- crates/accuracy/src/readout_caps/mod.rs | 101 +++++++----- crates/accuracy/src/readout_caps/tests.rs | 2 + .../stage1-pass2-reuse-candidates/src/lib.rs | 5 + .../src/readout_requirements.rs | 10 +- .../src/test_fixtures.rs | 1 + .../stage3-plan-selection/src/cost/tests.rs | 1 + .../src/milp/accuracy.rs | 152 +++++++++--------- .../src/milp/model/mod.rs | 7 +- .../src/milp/model/tests.rs | 17 +- .../milp/model/tests/union_bound_oracle.rs | 23 +-- crates/stage3-plan-selection/src/report.rs | 13 +- crates/types/src/accuracy/target/mod.rs | 20 +++ crates/types/src/accuracy/target/tests.rs | 17 ++ crates/types/src/stages/logical/memo/mod.rs | 4 + .../src/stages/logical/memo_candidates.rs | 31 ++++ .../stages/physical/memo_candidates/tests.rs | 3 + docs/design_docs/concepts/accuracy-models.md | 53 +++--- 20 files changed, 395 insertions(+), 160 deletions(-) diff --git a/crates/accuracy/src/estimate_dependence.rs b/crates/accuracy/src/estimate_dependence.rs index 5150e30c4..28d7192fe 100644 --- a/crates/accuracy/src/estimate_dependence.rs +++ b/crates/accuracy/src/estimate_dependence.rs @@ -10,12 +10,13 @@ //! operator copies each value from one input //! ([`takes_each_value_from_one_input`]), and on every value of any other //! input; only through inputs whose error reaches the output -//! ([`input_error_reaches_output`], which Stage 3 reads through -//! `ErrorMetricsOfMemo::input_error_reaches_output`). +//! ([`input_error_reaches_output`]) or whose values are chosen by +//! estimates ([`values_chosen_by_estimate`]); both read through +//! `ErrorMetricsOfMemo::input_estimates_reach_output`. use asap_types::ir::operator::arguments::BinaryOpKind; use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator}; -use asap_types::stages::MemoGroupId; +use asap_types::stages::{Memo, MemoGroupId}; use asap_types::summary_type::SketchStatistic; use crate::metric_rule::ErrorContractPerMetric; @@ -36,6 +37,40 @@ pub(crate) fn input_error_reaches_output( .any(|contract| contract.input_error_reaches_output(input)) } +/// Per group of `memo`, by id: whether which values its output holds is +/// chosen by estimates, so that a consumer keeping or dropping a value by +/// its labels depends on every estimate that choice read. A top-k readout +/// chooses its keys by all the estimates it ranks +/// ([`ranks_by_estimate`]), and an operator reading such an input value by +/// value ([`inputs_read_value_by_value`]) passes the choice on: `topk(3, +/// q) * 2` holds the 3 keys the top-k chose. A reduction does not: `sum` +/// returns its one value whichever keys it reads. `order` is the memo's +/// topological order (`Memo::topological_order`). +/// +/// Example: in `p90 and topk(3, q)`, the top-k's group and the `and`'s +/// are chosen by estimate, `p90`'s is not. +pub fn values_chosen_by_estimate(memo: &Memo, order: &[MemoGroupId]) -> Vec { + let mut chosen = vec![false; memo.groups().len()]; + for &group in order { + chosen[group.0 as usize] = + memo.groups()[group.0 as usize] + .candidates + .iter() + .any(|candidate| { + let operator = &candidate.expression.operator; + ranks_by_estimate(operator) + || inputs_read_value_by_value(operator).is_some_and(|value_by_value| { + operator + .children() + .iter() + .take(value_by_value) + .any(|child| chosen[child.0 as usize]) + }) + }); + } + chosen +} + /// Whether `operator`, a readout, chooses its output values by the /// estimates it computes: a top-k ranks every item of its state, so each /// of its output values depends on all of them. diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 6c78f3a4c..75dd89a41 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -39,4 +39,4 @@ pub use measured::{ MeasuredSketchAccuracyError, }; pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; -pub use readout_caps::{readout_levels, readout_value_counts_of_queries, ReadoutRequiredAccuracy}; +pub use readout_caps::{readout_levels, ReadoutLevels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index ce8dbdd7f..0d01e9782 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -12,6 +12,7 @@ use asap_types::ir::{QueryRoot, ScalarExpr}; use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId}; use crate::error_contract::{error_contract, ErrorContract}; +use crate::estimate_dependence::{takes_each_value_from_one_input, values_chosen_by_estimate}; use crate::readout::sketch_readout_error_metric; use crate::readout_caps::caps; @@ -33,6 +34,9 @@ pub struct ErrorMetricsOfMemo { /// not a sketch readout, or one whose summary type has no guarantee /// for the statistic it reads. readout_error_metric: Vec>>, + /// [group]: whether which values its output holds is chosen by + /// estimates ([`values_chosen_by_estimate`]). + values_chosen_by_estimate: Vec, } impl ErrorMetricsOfMemo { @@ -91,6 +95,7 @@ impl ErrorMetricsOfMemo { error_contract, error_metrics_allowed_at_output, readout_error_metric, + values_chosen_by_estimate: values_chosen_by_estimate(memo, order), } } @@ -104,19 +109,48 @@ impl ErrorMetricsOfMemo { .as_ref() } - /// Whether `candidate`'s input `input` can make its output miss: its - /// error reaches the output in some metric - /// ([`crate::estimate_dependence::input_error_reaches_output`], the - /// test Pass 2's failure-probability split counts by). - pub fn input_error_reaches_output( + /// Whether an estimate input `input` of `candidate` (of `memo`) + /// depends on can make the output miss, so the output's failure + /// probability counts it: the input's error reaches the output in some + /// metric ($\beta > 0$, + /// [`crate::estimate_dependence::input_error_reaches_output`]), or which + /// values the input holds is chosen by estimates + /// ([`values_chosen_by_estimate`]). Pass 2's failure-probability split + /// and Stage 3's constraint (K) both count by it. + /// + /// Example: in `p90 and r` only `p90`'s estimates count; in `p90 and + /// topk(3, q)` the top-k's do too, as they choose the keys `and` keeps. + pub fn input_estimates_reach_output( &self, + memo: &Memo, candidate: CandidateForMemoGroupId, input: usize, ) -> bool { - crate::estimate_dependence::input_error_reaches_output( - &self.error_contract[candidate.group.0 as usize][candidate.index_in_group], - input, - ) + let child = memo.candidate(candidate).expression.operator.children()[input]; + self.values_chosen_by_estimate[child.0 as usize] + || crate::estimate_dependence::input_error_reaches_output( + &self.error_contract[candidate.group.0 as usize][candidate.index_in_group], + input, + ) + } + + /// Whether each output value of `candidate` (of `memo`) depends on the + /// estimates of one input only: it copies each value from one input + /// ([`takes_each_value_from_one_input`]) and no input's values are + /// chosen by estimates. `q or r` takes `r`'s value only where `q` has + /// no such series, so with `q` a top-k that choice reads all of `q`'s + /// estimates. + pub fn each_value_depends_on_one_input( + &self, + memo: &Memo, + candidate: CandidateForMemoGroupId, + ) -> bool { + let operator = &memo.candidate(candidate).expression.operator; + takes_each_value_from_one_input(operator) + && !operator + .children() + .iter() + .any(|child| self.values_chosen_by_estimate[child.0 as usize]) } /// The error metric of `candidate` when it is a sketch readout. diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index de3c1329a..31264f6ee 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -61,6 +61,23 @@ pub struct ReadoutRequiredAccuracy { pub queries_without_per_value_failure_probability: Vec, } +/// What [`readout_levels`] gives: each readout's levels, and the readout +/// values each query's δ is split over. +#[derive(Debug, Clone, PartialEq)] +pub struct ReadoutLevels { + /// Per sketch readout without a requirement yet: its levels. + pub of_readout: BTreeMap, + /// Per query, by index: the readout values (estimates) its output can + /// depend on, `R` and the fan-in + /// ([`ReadoutsReadByQueries::readout_values`]), which name the bound + /// its δ is split by (`AccuracyTarget::failure_probability_bound`) and + /// Stage 3's constraint (K) checks; `None` for a query without a + /// target. + /// + /// Example: `q or r` over 100 series each: `R` 200, fan-in 1. + pub readout_values_of_query: Vec>, +} + /// The required-accuracy levels of the sketch readouts without a /// requirement yet, per readout ([`ReadoutRequiredAccuracy`]), from /// `error_metrics` of `memo` ([`ErrorMetricsOfMemo::of`]). `order` is the @@ -96,7 +113,7 @@ pub fn readout_levels( error_metrics: &ErrorMetricsOfMemo, order: &[MemoGroupId], evidence: &dyn AccuracyEvidence, -) -> BTreeMap { +) -> ReadoutLevels { let caps = caps(memo, &error_metrics.error_contract, order); let mut readouts: Vec<(CandidateForMemoGroupId, AccuracyMetric)> = Vec::new(); for (index, group) in memo.groups().iter().enumerate() { @@ -129,7 +146,10 @@ pub fn readout_levels( } let mut levels_of_group: HashMap<(MemoGroupId, AccuracyMetric), ReadoutRequiredAccuracy> = HashMap::new(); - readouts + let readout_values_of_query = (0..memo.queries().len()) + .map(|query| readouts_read.readout_values.get(&query).copied()) + .collect(); + let of_readout = readouts .into_iter() .map(|(readout, metric)| { let group = readout.group; @@ -148,7 +168,11 @@ pub fn readout_levels( .clone(); (readout, levels) }) - .collect() + .collect(); + ReadoutLevels { + of_readout, + readout_values_of_query, + } } /// The levels of readout group `group` in `metric` ([`readout_levels`]). @@ -409,16 +433,20 @@ struct ReadoutsReadByQueries { /// single level of its cap. Readouts of one value by different summary /// types are alternatives in one group's candidates, not a combination. composed: BTreeSet, - /// Per query, by index: the readout values its output can depend on - /// through β > 0, among alternative candidates of a group the largest. - /// Per evaluation, summed over the readout groups below it - /// (`Memo::max_values_per_evaluation` each); per output value, 1 at a - /// readout (all of its values for a top-k, which ranks them), carried - /// value by value through [`inputs_read_value_by_value`] operators - /// (summed over inputs, or the largest for `or` and a concatenation) - /// and all of the inputs' per-evaluation values through any other - /// operator (a reduction); a readout group also counts its other - /// candidates (a reduction over readouts below it). `max by (job) + /// Per query, by index: the readout values (estimates) its output can + /// depend on, through inputs whose estimates reach their consumer's + /// output (`ErrorMetricsOfMemo::input_estimates_reach_output`: β > 0, + /// or values chosen by estimates), among alternative candidates of a + /// group the largest. Per evaluation, summed over the readout groups + /// below it (`Memo::max_estimates_per_evaluation` each: its values, or + /// every item a top-k ranks); per output value, 1 at a readout (all of + /// its estimates for a top-k, which ranks them), carried value by + /// value through [`inputs_read_value_by_value`] operators (summed over + /// inputs, or the largest where each value depends on one input, + /// `ErrorMetricsOfMemo::each_value_depends_on_one_input`: `or` and a + /// concatenation) and all of the inputs' per-evaluation values through + /// any other operator (a reduction); a readout group also counts its + /// other candidates (a reduction over readouts below it). `max by (job) /// (q{host="h0"} or q{host="h1"})` with 3 series per side: 6 and 6; /// `q or r` alone: 6 and 1; `q / count(r)`: `q`'s alone. A readout /// group reached on two paths (`q or q`) counts twice, which only @@ -427,7 +455,8 @@ struct ReadoutsReadByQueries { } /// The [`ReadoutsReadByQueries`] of `memo`, in one walk up `order` (the -/// memo's topological order) from the readout groups `readout_groups`. +/// memo's topological order) from its readout groups (those with a +/// candidate that is a sketch readout, `ErrorMetricsOfMemo::readout_error_metric`). fn readouts_read_by_queries( memo: &Memo, error_metrics: &ErrorMetricsOfMemo, @@ -462,21 +491,32 @@ fn readouts_read_by_queries( let candidates = &memo.groups()[group.0 as usize].candidates; let mut most_inputs_reading_with_error = 0; let mut most_readout_values = none_read; - for (candidate, contract_per_metric) in - candidates.iter().zip(&error_contract[group.0 as usize]) + for (index_in_group, (candidate, contract_per_metric)) in candidates + .iter() + .zip(&error_contract[group.0 as usize]) + .enumerate() { + let id = CandidateForMemoGroupId { + group, + index_in_group, + }; let operator = &candidate.expression.operator; let children = operator.children(); - let takes_each_value_from_one_input = takes_each_value_from_one_input(operator); + let reads_readouts = |input: usize| { + readout_values_per_group[children[input].0 as usize].per_evaluation != Some(0) + }; let reads_with_error = |input: &usize| { - readout_values_per_group[children[*input].0 as usize].per_evaluation != Some(0) - && input_error_reaches_output(contract_per_metric, *input) + reads_readouts(*input) && input_error_reaches_output(contract_per_metric, *input) + }; + let reads_estimates = |input: &usize| { + reads_readouts(*input) + && error_metrics.input_estimates_reach_output(memo, id, *input) }; let inputs_reading_with_error = (0..children.len()).filter(reads_with_error).count(); // Each output value comes from one input, so the output's error // is one input's, never a sum of several; its values still // depend on the readouts of every input. - let inputs_combined = if takes_each_value_from_one_input { + let inputs_combined = if takes_each_value_from_one_input(operator) { inputs_reading_with_error.min(1) } else { inputs_reading_with_error @@ -484,7 +524,7 @@ fn readouts_read_by_queries( most_inputs_reading_with_error = most_inputs_reading_with_error.max(inputs_combined); let read_of = |input: usize| readout_values_per_group[children[input].0 as usize]; let per_evaluation = (0..children.len()) - .filter(reads_with_error) + .filter(reads_estimates) .try_fold(0u64, |total, input| { total.checked_add(read_of(input).per_evaluation?) }); @@ -492,14 +532,14 @@ fn readouts_read_by_queries( // `value_by_value` inputs, and every value of any other input. let per_output_value = inputs_read_value_by_value(operator).map_or(per_evaluation, |value_by_value| { - let mut per_input = (0..children.len()).filter(reads_with_error).map(|input| { + let mut per_input = (0..children.len()).filter(reads_estimates).map(|input| { if input < value_by_value { read_of(input).per_output_value } else { read_of(input).per_evaluation } }); - if takes_each_value_from_one_input { + if error_metrics.each_value_depends_on_one_input(memo, id) { per_input.try_fold(0u64, |most, values| Some(most.max(values?))) } else { per_input.try_fold(0u64, |total, values| total.checked_add(values?)) @@ -553,21 +593,6 @@ fn readouts_read_by_queries( } } -/// Per query with a target, by index: the readout values (estimates) its -/// output can depend on per evaluation, `R` and the fan-in -/// ([`ReadoutsReadByQueries::readout_values`]), which -/// `AccuracyTarget::failure_probability_bound` chooses the query's bound -/// from. Pass 2 splits δ by it, and Stage 3's constraint (K) checks the -/// same bound, so the two cannot choose differently. `order` is the memo's -/// topological order (`Memo::topological_order`). -pub fn readout_value_counts_of_queries( - memo: &Memo, - error_metrics: &ErrorMetricsOfMemo, - order: &[MemoGroupId], -) -> BTreeMap { - readouts_read_by_queries(memo, error_metrics, order).readout_values -} - /// Keeps the larger of `cap` and the known cap of `query` in `metric`. fn raise( caps: &mut HashMap>, diff --git a/crates/accuracy/src/readout_caps/tests.rs b/crates/accuracy/src/readout_caps/tests.rs index bcd0ec73e..ed846c6f1 100644 --- a/crates/accuracy/src/readout_caps/tests.rs +++ b/crates/accuracy/src/readout_caps/tests.rs @@ -23,6 +23,7 @@ fn levels_of(memo: &Memo, readout: MemoGroupId) -> Vec { &order, &NoMeasuredGuarantees, ) + .of_readout .remove(&own_candidate(readout)) .expect("a sketch readout") .levels @@ -337,6 +338,7 @@ fn failure_probability_of(memo: &Memo, readout: MemoGroupId) -> Option { &order, &NoMeasuredGuarantees, ) + .of_readout .remove(&own_candidate(readout)) .expect("a sketch readout"); if !required diff --git a/crates/stage1-pass2-reuse-candidates/src/lib.rs b/crates/stage1-pass2-reuse-candidates/src/lib.rs index 618059393..3b6f942f7 100644 --- a/crates/stage1-pass2-reuse-candidates/src/lib.rs +++ b/crates/stage1-pass2-reuse-candidates/src/lib.rs @@ -70,6 +70,10 @@ pub enum Pass2RuleError { /// groups each query may read only exactly /// (`ErrorMetricsOfMemo::exact_only_groups_of_query`). /// +/// Step 3 also records the readout values each query's δ is split over, +/// whose bound Stage 3 checks +/// (`LogicalASAPMemoCandidates::readout_values_of_query`). +/// /// `entries[i]` is query `i`'s workload entry (its evaluation grid). pub fn generate_logical_asap_memo_candidates( pass1: Memo, @@ -151,5 +155,6 @@ pub fn generate_logical_asap_memo_candidates( pruned, evaluation_grid_of_group, exact_only_groups_of_query, + readout_values_of_query: readout_requirements.readout_values_of_query, }) } diff --git a/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs index 256ffbb5e..11a7c9df5 100644 --- a/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs +++ b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs @@ -7,6 +7,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::Pass2RuleError; use asap_accuracy::{readout_levels, AccuracyEvidence, ErrorMetricsOfMemo}; +use asap_types::accuracy::ReadoutValueCounts; use asap_types::stages::{ CandidateForMemoGroupId, LogicalRequirements, Memo, MemoGroupId, RequiredAccuracy, }; @@ -23,6 +24,9 @@ pub(crate) struct ReadoutRequirementsOutcome { /// failure probability, which it may read only exactly (another /// query's level says nothing of its own δ). pub(crate) exact_only_readout_groups_of_query: BTreeMap>, + /// Per query, by index: the readout values its δ is split over + /// (`ReadoutLevels::readout_values_of_query`). + pub(crate) readout_values_of_query: Vec>, } /// Gives every sketch readout (`SummaryEstimate` over a sketch state) its @@ -37,12 +41,14 @@ pub(crate) fn set_readout_requirements( order: &[MemoGroupId], evidence: &dyn AccuracyEvidence, ) -> Result { + let error_metrics = ErrorMetricsOfMemo::of(memo, order); + let levels = readout_levels(memo, &error_metrics, order, evidence); let mut outcome = ReadoutRequirementsOutcome { readouts_without_per_value_failure_probability: BTreeMap::new(), exact_only_readout_groups_of_query: BTreeMap::new(), + readout_values_of_query: levels.readout_values_of_query, }; - let error_metrics = ErrorMetricsOfMemo::of(memo, order); - for (readout, required) in readout_levels(memo, &error_metrics, order, evidence) { + for (readout, required) in levels.of_readout { for &query in &required.queries_without_per_value_failure_probability { outcome .exact_only_readout_groups_of_query diff --git a/crates/stage2-physical-optimization/src/test_fixtures.rs b/crates/stage2-physical-optimization/src/test_fixtures.rs index feb34a87a..edd9756e0 100644 --- a/crates/stage2-physical-optimization/src/test_fixtures.rs +++ b/crates/stage2-physical-optimization/src/test_fixtures.rs @@ -93,6 +93,7 @@ pub fn logical( ) -> LogicalASAPMemoCandidates { LogicalASAPMemoCandidates { exact_only_groups_of_query: vec![BTreeSet::new(); memo.queries().len()], + readout_values_of_query: vec![None; memo.queries().len()], memo, edges, failing_logical_edges: vec![], diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs index 15dd5564e..8033808a3 100644 --- a/crates/stage3-plan-selection/src/cost/tests.rs +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -152,6 +152,7 @@ fn no_execution_reading_a_producer_costs_less_than_nothing() { pruned: vec![], evaluation_grid_of_group: vec![None; groups], exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, candidates, edges, diff --git a/crates/stage3-plan-selection/src/milp/accuracy.rs b/crates/stage3-plan-selection/src/milp/accuracy.rs index e6d183a67..48c808032 100644 --- a/crates/stage3-plan-selection/src/milp/accuracy.rs +++ b/crates/stage3-plan-selection/src/milp/accuracy.rs @@ -16,8 +16,7 @@ use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Va use asap_accuracy::{ inputs_read_value_by_value, query_roots_with_target, ranks_by_estimate, - readout_value_counts_of_queries, takes_each_value_from_one_input, tolerated_limit, - ErrorContract, ErrorMetricsOfMemo, + takes_each_value_from_one_input, tolerated_limit, ErrorContract, ErrorMetricsOfMemo, }; use asap_types::accuracy::{AccuracyMetric, FailureProbabilityBound}; use asap_types::stages::{CandidateForMemoGroupId, MemoGroupId}; @@ -31,7 +30,7 @@ use crate::selectable::QueryId; /// summed δ of estimates (one readout value in one evaluation) by the /// union bound (`accuracy-models.md`, "Which estimates a result depends /// on"). A failure probability is `None` when no declared fact bounds it -/// (a readout whose values per evaluation are unknown). +/// (a readout whose estimates per evaluation are unknown). /// /// Example: a DDSketch readout over 1000 series required at relative 0.01 /// with δ = 1e-3 is 0.01 in `Relative`, 0 in every other metric, 1e-3 for @@ -118,19 +117,30 @@ impl ExecutionError { } /// How an execution's output values read its inputs, which decides how -/// (K) composes its failure probabilities (`asap_accuracy`'s operator -/// classes, the ones Pass 2's split counts by). -#[derive(Debug, Clone, Copy)] +/// (K) composes its error and failure probabilities (`asap_accuracy`'s +/// operator classes, the ones Pass 2's split counts by). +#[derive(Debug, Clone)] struct HowInputsAreRead { /// Its first so many inputs are read value by value: each output value /// reads one value of each ([`inputs_read_value_by_value`]); every /// other input is read whole. `None`: all are read whole (a reduction). value_by_value: Option, /// Each output value is copied from one input (`or`, a concatenation: - /// [`takes_each_value_from_one_input`]), so the inputs read value by - /// value add the largest of their one-value failure probabilities, not - /// their sum. + /// [`takes_each_value_from_one_input`]), so its error is the largest of + /// the inputs', not their sum. each_value_from_one_input: bool, + /// Each output value depends on the estimates of one input only + /// (`ErrorMetricsOfMemo::each_value_depends_on_one_input`): the inputs + /// read value by value add the largest of their one-value failure + /// probabilities, not their sum. Not for `topk(3, q) or r`, whose + /// values from `r` depend on the top-k's choice too. + each_value_depends_on_one_input: bool, + /// Per input: whether an estimate it depends on can make the output + /// miss (`ErrorMetricsOfMemo::input_estimates_reach_output`), so its + /// failure probabilities add to the output's. Not a count's input, nor + /// the right side of `and` and `unless`, unless a top-k chooses its + /// values. + estimates_reach_output: Vec, } impl HowInputsAreRead { @@ -138,6 +148,12 @@ impl HowInputsAreRead { fn reads_value_by_value(&self, input: usize) -> bool { self.value_by_value.is_some_and(|count| input < count) } + + /// Whether an estimate input `input` depends on can make the output + /// miss ([`Self::estimates_reach_output`]). + fn estimates_of_input_reach_output(&self, input: usize) -> bool { + self.estimates_reach_output[input] + } } /// What composing one execution's error from what its inputs read gives @@ -166,8 +182,8 @@ pub(super) struct AccuracyRules { /// Per execution, the error it adds: a readout's requirement /// (`CandidateForMemoGroup::required_accuracy`), its δ for one value /// (`R` times it for a top-k, which ranks every estimate) and `R` - /// times its δ for all values (`Memo::max_estimates_per_evaluation`); - /// 0 for any other. + /// times its δ for all values (`Memo::max_estimates_per_evaluation`; + /// 0 at δ = 0 whatever `R`); 0 for any other. own_error_per_execution: Vec, /// Per execution, how its output values read its inputs. how_inputs_are_read: Vec, @@ -183,16 +199,15 @@ pub(super) struct AccuracyRules { roots: HashMap<(QueryId, MemoGroupId), RootLimits>, } -/// What a root check of (K) bounds one query output by, each limit -/// already widened by [`tolerated_limit`] as the edge check widens it. -/// Two outputs with equal limits have the same root rows, so they compare -/// and hash by the limits' bits. +/// What a root check of (K) bounds one query output by. The limits are +/// widened by [`tolerated_limit`] where they are compared, as the edge +/// check widens them. Two outputs with equal limits have the same root +/// rows, so they compare and hash by the limits' bits. /// /// Example: `p99 / p50` at relative 0.03 with δ = 0.01 over a whole result /// of 100 values, reading two readouts over 100 series each, allows 0.03 -/// (widened) in `Relative`, 0 in every other metric, and checks each -/// covered value: its one-value failure probability at most 1e-4 -/// (widened). +/// in `Relative`, 0 in every other metric, and checks each covered value: +/// its one-value failure probability at most 1e-4. #[derive(Debug, Clone, Copy)] pub(super) struct RootLimits { /// Per metric (by [`AccuracyMetric::index`]), the error the output may @@ -200,25 +215,25 @@ pub(super) struct RootLimits { /// ([`query_roots_with_target`]), 0 in any other. tolerated_error_per_metric: [f64; AccuracyMetric::COUNT], /// The union bound the query's failure probability is checked by, with - /// its limit (`AccuracyTarget::failure_probability_bound`, widened by - /// [`tolerated_limit`] where it is compared; - /// from the readout counts Pass 2 splits δ by); `None` without a δ, - /// or with neither bound known (Pass 2 then gives the query's readouts - /// no level, and (X) makes it read them exactly). + /// its limit (the bound Pass 2 split δ by, + /// `LogicalASAPMemoCandidates::failure_probability_bound`); + /// `None` without a δ, or with neither bound known (Pass 2 then gives + /// the query's readouts no level, and (X) makes it read them exactly). failure_probability_bound: Option, } impl RootLimits { - /// The limits' bits, which equality, hashing and order compare. - fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option<(u64, u64)>) { + /// The limits' bits, which equality, hashing and order compare: the + /// error per metric, and the bound's kind and limit (the rows read no + /// more of it). + fn bits(&self) -> ([u64; AccuracyMetric::COUNT], Option<(bool, u64)>) { ( self.tolerated_error_per_metric.map(f64::to_bits), - self.failure_probability_bound.map(|bound| match bound { - FailureProbabilityBound::EveryEstimateRead { limit } => (0, limit.to_bits()), - FailureProbabilityBound::EachCoveredValue { - values_covered, - limit, - } => (values_covered.max(1), limit.to_bits()), + self.failure_probability_bound.map(|bound| { + ( + matches!(bound, FailureProbabilityBound::EachCoveredValue { .. }), + bound.limit().to_bits(), + ) }), ) } @@ -283,9 +298,13 @@ impl AccuracyRules { if let Some(required) = candidate.required_accuracy() { own.error_bound_per_metric[required.metric.index()] = required.max_error; let delta = required.failure_probability; - let of_every_estimate = memo - .max_estimates_per_evaluation(id.group) - .map(|estimates| estimates as f64 * delta); + // At δ = 0 no estimate can fail, however many there are. + let of_every_estimate = if delta == 0.0 { + Some(0.0) + } else { + memo.max_estimates_per_evaluation(id.group) + .map(|estimates| estimates as f64 * delta) + }; own.failure_probability_of_one_value = if ranks_by_estimate(&candidate.expression.operator) { of_every_estimate @@ -297,6 +316,7 @@ impl AccuracyRules { own }) .collect(); + let error_metrics = ErrorMetricsOfMemo::of(memo, &order); let how_inputs_are_read = candidate_of_execution .iter() .map(|&id| { @@ -304,6 +324,11 @@ impl AccuracyRules { HowInputsAreRead { value_by_value: inputs_read_value_by_value(operator), each_value_from_one_input: takes_each_value_from_one_input(operator), + each_value_depends_on_one_input: error_metrics + .each_value_depends_on_one_input(memo, id), + estimates_reach_output: (0..operator.children().len()) + .map(|input| error_metrics.input_estimates_reach_output(memo, id, input)) + .collect(), } }) .collect(); @@ -324,17 +349,11 @@ impl AccuracyRules { (0..candidate_of_execution.len()).collect(); executions_producers_first .sort_by_key(|&c| position_in_order[candidate_of_execution[c].group.0 as usize]); - let error_metrics = ErrorMetricsOfMemo::of(memo, &order); - let readout_values = readout_value_counts_of_queries(memo, &error_metrics, &order); let roots = query_roots_with_target(memo) .into_iter() .map(|(query, root, target, allowed_metrics)| { - let failure_probability_bound = readout_values.get(&query).and_then(|&counts| { - target.failure_probability_bound( - memo.queries()[query].number_of_values_per_query_evaluation, - counts, - ) - }); + let failure_probability_bound = + physical.logical_candidates.failure_probability_bound(query); let mut tolerated_error_per_metric = [0.0; AccuracyMetric::COUNT]; for metric in allowed_metrics { tolerated_error_per_metric[metric.index()] = tolerated_limit(target.max_error); @@ -364,18 +383,6 @@ impl AccuracyRules { self.roots.get(&(query, group)) } - /// The union bound `query`'s failure probability is checked by, with its - /// limit; `None` without one. - pub(super) fn failure_probability_bound( - &self, - query: QueryId, - ) -> Option { - self.roots - .iter() - .find(|((root_query, _), _)| *root_query == query) - .and_then(|(_, limits)| limits.failure_probability_bound) - } - /// Execution `c`'s error contract in metric `m` (by /// [`AccuracyMetric::index`]); `None` where it has none (the metric /// rule). @@ -422,15 +429,12 @@ impl AccuracyRules { error: own, inputs_admitted: true, }; - let how = self.how_inputs_are_read[c]; + let how = &self.how_inputs_are_read[c]; let mut one_value_of_inputs_read_value_by_value = Vec::new(); let mut all_values_of_inputs_read_whole = Vec::new(); let mut all_values_of_inputs = Vec::new(); for (i, read) in inputs_read { - if self - .error_metrics - .input_error_reaches_output(self.candidate_of_execution[c], i) - { + if how.estimates_of_input_reach_output(i) { all_values_of_inputs.push(read.failure_probability_of_all_values); if how.reads_value_by_value(i) { one_value_of_inputs_read_value_by_value @@ -465,7 +469,7 @@ impl AccuracyRules { } } } - let of_inputs_read_value_by_value = if how.each_value_from_one_input { + let of_inputs_read_value_by_value = if how.each_value_depends_on_one_input { largest_of(one_value_of_inputs_read_value_by_value) } else { sum_of(one_value_of_inputs_read_value_by_value) @@ -814,12 +818,9 @@ impl<'a> ErrorModel<'a> { if !self.has_rows[c] || !(one_value || all_values) { continue; } - let how = rules.how_inputs_are_read[c]; + let how = &rules.how_inputs_are_read[c]; for (i, edges) in self.selectable_edges_per_input[c].iter().enumerate() { - if !rules - .error_metrics - .input_error_reaches_output(rules.candidate_of_execution[c], i) - { + if !how.estimates_of_input_reach_output(i) { continue; } let value_by_value = how.reads_value_by_value(i); @@ -1002,7 +1003,7 @@ pub(super) fn accuracy_constraints( // inputs read whole, and Σ_i HW_{c,i}. let has_one_value = accuracy.failure_probability_of_one_value.contains_key(&c); let has_all_values = accuracy.failure_probability_of_all_values.contains_key(&c); - let how = rules.how_inputs_are_read[c]; + let how = &rules.how_inputs_are_read[c]; let mut one_value_of_inputs_read_value_by_value: Vec = Vec::new(); let mut all_values_of_inputs_read_whole = Expression::default(); let mut all_values_of_inputs = Expression::default(); @@ -1039,12 +1040,10 @@ pub(super) fn accuracy_constraints( } error_of_input.push(contract.error_factor_per_input[i] * of_input); } - // Only an input whose error reaches the output can make it - // miss: a count's input and the right of `and` add nothing. - if !rules - .error_metrics - .input_error_reaches_output(rules.candidate_of_execution[c], i) - { + // Only an input whose estimates reach the output can make it + // miss: a count's input and the right of `and` add nothing, + // unless a top-k chooses their values. + if !how.estimates_of_input_reach_output(i) { continue; } let value_by_value = how.reads_value_by_value(i); @@ -1104,7 +1103,8 @@ pub(super) fn accuracy_constraints( .failure_probability_of_one_value .expect("a bounded ceiling has a bounded own term"); let whole = all_values_of_inputs_read_whole; - if how.each_value_from_one_input && !one_value_of_inputs_read_value_by_value.is_empty() + if how.each_value_depends_on_one_input + && !one_value_of_inputs_read_value_by_value.is_empty() { for of_input in one_value_of_inputs_read_value_by_value { constraints.push(constraint!(of_one_value >= own + of_input + whole.clone())); @@ -1124,6 +1124,10 @@ pub(super) fn accuracy_constraints( } let memo = &inputs.physical.logical_candidates.memo; let output_groups = memo.output_groups(); + let mut outputs_per_query = vec![0usize; memo.queries().len()]; + for &(query, _) in &output_groups { + outputs_per_query[query] += 1; + } // Per query with a bound and several outputs: the bound and, per // output, what its answering execution gives. let mut several_roots: BTreeMap = @@ -1138,7 +1142,7 @@ pub(super) fn accuracy_constraints( let Some(limits) = rules.root_limits(*query_id, group) else { continue; }; - let outputs_of_query = output_groups.iter().filter(|(q, _)| *q == query).count(); + let outputs_of_query = outputs_per_query[query]; let mut checked_per_execution: Vec<(Variable, ErrorTerm)> = Vec::new(); for &v in executions { let z = z[&(*query_id, v)]; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 1f449d8bd..ebf47518b 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -561,9 +561,10 @@ pub(crate) fn solve_selection( let mut missed_at_default_tolerance: Option = None; let below_solver_precision = |query: QueryId| SelectionMilpError::TargetBelowSolverPrecision { query, - failure_probability_bound: rules - .as_ref() - .and_then(|rules| rules.failure_probability_bound(query)), + failure_probability_bound: inputs + .physical + .logical_candidates + .failure_probability_bound(query.0), }; let (selected, selected_edges, answering) = loop { let started = Instant::now(); diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index d67dce47c..50fe3f8fd 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -47,6 +47,7 @@ use crate::milp::accuracy::{AccuracyRules, AccuracyVariables, ErrorModel}; use crate::plan::plan_of; use crate::retention::{edge_retention_need_ms, least_retention_ms_per_execution}; use crate::selectable::{QueryId, SelectableEdge}; +use asap_accuracy::{readout_levels, ErrorMetricsOfMemo, NoMeasuredGuarantees}; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); @@ -70,7 +71,8 @@ fn empty_memo() -> Memo { /// Stage 2's output over `memo`, with one query-time candidate per /// (logical candidate, lifetime) of `candidates` and every edge between -/// a candidate of an input group and its consumer admissible. +/// a candidate of an input group and its consumer admissible, and each +/// query's failure-probability bound as Pass 2 records it. fn physical( memo: Memo, candidates: &[(CandidateForMemoGroupId, Lifetime)], @@ -108,6 +110,18 @@ fn physical( } } let groups = memo.groups().len(); + let readout_values_of_query = memo + .topological_order() + .map(|order| { + readout_levels( + &memo, + &ErrorMetricsOfMemo::of(&memo, &order), + &order, + &NoMeasuredGuarantees, + ) + .readout_values_of_query + }) + .unwrap_or_default(); PhysicalASAPMemoCandidates::new( LogicalASAPMemoCandidates { memo, @@ -116,6 +130,7 @@ fn physical( pruned: vec![], evaluation_grid_of_group: vec![None; groups], exact_only_groups_of_query: vec![], + readout_values_of_query, }, physical_candidates, edges, diff --git a/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs b/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs index fae29e0bc..d3c8a460e 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests/union_bound_oracle.rs @@ -24,7 +24,7 @@ //! //! Each readout group gets the level Pass 2's split gives each query //! reaching it (`AccuracyTarget::failure_probability_per_readout_value` -//! over `asap_accuracy::readout_value_counts_of_queries`) and random +//! over the counts `asap_accuracy::readout_levels` records) and random //! further copies, some looser and cheaper, so the solver is tempted by //! copies given for another query. Queries are planned alone and in pairs //! (a per-value query on a readout with a query over a shape reading it), @@ -32,7 +32,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use asap_accuracy::{readout_value_counts_of_queries, ErrorMetricsOfMemo}; +use asap_accuracy::{readout_levels, ErrorMetricsOfMemo, NoMeasuredGuarantees}; use asap_types::workload::bounds::source_name; use super::*; @@ -439,9 +439,18 @@ fn check( // Pass 2's level per query reaching each readout group, plus random // copies at δ from 1e-5 to 0.5. let order = memo.topological_order().unwrap(); - let counts = - readout_value_counts_of_queries(&memo, &ErrorMetricsOfMemo::of(&memo, &order), &order); + let readout_values_of_query = readout_levels( + &memo, + &ErrorMetricsOfMemo::of(&memo, &order), + &order, + &NoMeasuredGuarantees, + ) + .readout_values_of_query; for &readout in &readouts { + let pass1 = CandidateForMemoGroupId { + group: readout, + index_in_group: 0, + }; let mut levels: Vec = memo.groups()[readout.0 as usize] .reached_by_queries() .iter() @@ -452,17 +461,13 @@ fn check( .as_ref()? .failure_probability_per_readout_value( memo_query.number_of_values_per_query_evaluation, - counts[&query], + readout_values_of_query[query]?, ) }) .collect(); for _ in 0..2 { levels.push([1e-5, 1e-4, 1e-3, 5e-3, 0.01, 0.05, 0.5][rng.below(7) as usize]); } - let pass1 = CandidateForMemoGroupId { - group: readout, - index_in_group: 0, - }; let unset = memo.candidate(pass1).clone(); let requirement = |failure_probability| { vec![LogicalRequirements { diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index aca48c406..da538707e 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -226,7 +226,8 @@ pub enum ErrorSelectingPlan { /// failure probability of 1e-12 for each covered value (δ = 0.01 over /// 1e10 values), below the solver's feasibility tolerance 1e-10. #[error( - "query {query}: an accuracy target is below the solver's precision (failure probability bound {failure_probability_bound:?}); the solver's plan misses it, so no plan is returned" + "query {query}: an accuracy target is below the solver's precision ({}); the solver's plan misses it, so no plan is returned", + failure_probability_bound_in_words(.failure_probability_bound) )] AccuracyTargetBelowSolverPrecision { query: usize, @@ -237,6 +238,16 @@ pub enum ErrorSelectingPlan { UnsupportedCostModel(String), } +/// `bound` in words for [`ErrorSelectingPlan::AccuracyTargetBelowSolverPrecision`]: +/// "failure probability: each of 100 covered values fails with at most +/// 0.0001", or "no failure probability bound" without one. +fn failure_probability_bound_in_words(bound: &Option) -> String { + match bound { + Some(bound) => format!("failure probability: {bound}"), + None => "no failure probability bound".to_string(), + } +} + /// The selection MILP is infeasible, with the reasons executions could not /// be costed. pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { diff --git a/crates/types/src/accuracy/target/mod.rs b/crates/types/src/accuracy/target/mod.rs index b16c8ab25..85067ba33 100644 --- a/crates/types/src/accuracy/target/mod.rs +++ b/crates/types/src/accuracy/target/mod.rs @@ -408,6 +408,26 @@ pub enum FailureProbabilityBound { EachCoveredValue { values_covered: u64, limit: f64 }, } +/// The bound in words, for error messages: "every estimate read fails +/// with at most 0.01 in all" or "each of 100 covered values fails with at +/// most 0.0001". +impl std::fmt::Display for FailureProbabilityBound { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::EveryEstimateRead { limit } => { + write!(f, "every estimate read fails with at most {limit} in all") + } + Self::EachCoveredValue { + values_covered, + limit, + } => write!( + f, + "each of {values_covered} covered values fails with at most {limit}" + ), + } + } +} + impl FailureProbabilityBound { /// Its limit. pub fn limit(&self) -> f64 { diff --git a/crates/types/src/accuracy/target/tests.rs b/crates/types/src/accuracy/target/tests.rs index 0e1f213a2..383c0a5fb 100644 --- a/crates/types/src/accuracy/target/tests.rs +++ b/crates/types/src/accuracy/target/tests.rs @@ -560,3 +560,20 @@ fn readout_value_counts_combine_count_by_count() { counts(None, Some(2)) ); } + +/// A bound reads as words in error messages, not as its `Debug` form. +#[test] +fn a_failure_probability_bound_reads_as_words() { + assert_eq!( + FailureProbabilityBound::EveryEstimateRead { limit: 0.01 }.to_string(), + "every estimate read fails with at most 0.01 in all" + ); + assert_eq!( + FailureProbabilityBound::EachCoveredValue { + values_covered: 100, + limit: 1e-4 + } + .to_string(), + "each of 100 covered values fails with at most 0.0001" + ); +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 7538c9a15..020177575 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -187,6 +187,10 @@ impl Memo { /// series) its values are bounded by, as each item comes from a row; /// the smaller known one. This is the `R` a readout's failure /// probability is charged for when an output reads all its values. + /// The distinct-item bound is the deployment's contract, as HLL's + /// error is: a source declaring fewer distinct items than a state + /// counts gives too small an `R`, and a δ split over too few + /// estimates. /// /// Example: `topk(10, sum by (dst) (count_over_time(flows[5m])))` as /// one heap per `job`, with 3 jobs and at most 500 distinct `dst` per diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 34711cabc..e430af641 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -4,6 +4,7 @@ use std::collections::BTreeSet; +use crate::accuracy::{FailureProbabilityBound, ReadoutValueCounts}; use crate::stages::{ CandidateForMemoGroupId, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate, RootPruneReason, @@ -66,9 +67,39 @@ pub struct LogicalASAPMemoCandidates { /// (its MILP's constraint (X)); Pass 2 only lists it. Empty for a query without an accuracy /// target. pub exact_only_groups_of_query: Vec>, + /// Indexed by query: the readout values (estimates) its output can + /// depend on, `R` per evaluation and the fan-in per output value, as + /// Pass 2 counted them to split the query's δ over its readouts; + /// `None` for a query without an accuracy target. Stage 3's constraint + /// (K) checks the bound they name + /// ([`Self::failure_probability_bound`]), so the split and the check + /// cannot choose differently. Example: `max(quantile_over_time(0.9, + /// m[5m]))` over 1000 series: `R` 1000, fan-in 1000. + pub readout_values_of_query: Vec>, } impl LogicalASAPMemoCandidates { + /// The union bound query `query`'s failure probability is checked by, + /// with its limit: its target's + /// (`AccuracyTarget::failure_probability_bound`) over the readout + /// values Pass 2 counted for it ([`Self::readout_values_of_query`]). + /// `None` without a target δ, or with neither bound known (the query + /// then reads its readouts only exactly). + /// + /// Example: δ = 0.01 over the whole result of + /// `max(quantile_over_time(0.9, m[5m]))` over 1000 series: every + /// estimate read, at most 0.01. + pub fn failure_probability_bound(&self, query: usize) -> Option { + let memo_query = &self.memo.queries()[query]; + memo_query + .accuracy_target + .as_ref()? + .failure_probability_bound( + memo_query.number_of_values_per_query_evaluation, + self.readout_values_of_query.get(query).copied().flatten()?, + ) + } + /// Whether query `query` must not read `candidate`: it is not exact /// (it has sketch parameters θ) and its group is one the query may read /// only exactly ([`Self::exact_only_groups_of_query`]). A readout's own diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 622211238..1af97abe8 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -133,6 +133,7 @@ fn candidates( pruned: vec![], evaluation_grid_of_group, exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, vec![], vec![], @@ -692,6 +693,7 @@ fn new_indexes_only_admissible_edges() { pruned: vec![], evaluation_grid_of_group: vec![None; 5], exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, physical_candidates, edges.clone(), @@ -783,6 +785,7 @@ fn a_stored_chain_and_an_ingestion_time_result_are_stored_on_their_refresh_windo pruned: vec![], evaluation_grid_of_group: vec![grid_60s; group_count], exact_only_groups_of_query: vec![], + readout_values_of_query: vec![], }, vec![ candidate(series, timing, Lifetime::None), diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 0936b48fe..e354e511d 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -481,9 +481,10 @@ execution `v` answering query `q`: holds only up to `b` (a ratio's denominator). - Failure probabilities, with `HP_{c,i} ≥ P_u − M^P_u(1 − y_{u,c,i})` and `HW_{c,i} ≥ W_u − M^W_u(1 − y_{u,c,i})` per selectable producer `u`, over - only the inputs whose error reaches the output (`β > 0` in a metric of - `c`'s contract, the test Pass 2's split uses; a count's input and the - right input of `and` and `unless` add nothing): + only the inputs whose estimates reach the output (`β > 0` in a metric of + `c`'s contract, or values chosen by estimates, the test Pass 2's split + uses; a count's input and the right input of `and` and `unless` add + nothing unless a top-k chooses their values): - a readout: `P_c = δ_c` and `W_c = R_c δ_c`, with `δ_c` its required failure probability and `R_c` its estimates per evaluation (`Memo::max_estimates_per_evaluation`); a top-k readout, whose output @@ -491,9 +492,10 @@ execution `v` answering query `q`: Constants, as a readout's error is; - any other execution: `P_c ≥ Σ_{i read value by value} HP_{c,i} + Σ_{i read whole} HW_{c,i}`, - where an operator that copies each output value from one input (`or`, a - concatenation) has instead `P_c ≥ HP_{c,i} + Σ_{i read whole} HW_{c,i}` - for each input `i` read value by value; and `W_c ≥ Σ_i HW_{c,i}`. The + where an operator each of whose output values depends on one input's + estimates (`or`, a concatenation, with no top-k choosing an input's + values) has instead `P_c ≥ HP_{c,i} + Σ_{i read whole} HW_{c,i}` for + each input `i` read value by value; and `W_c ≥ Σ_i HW_{c,i}`. The operator classes are in the table below. - At the outputs of a query `q` with a target: `E_{v,m} ≤ ε_q + M(1 − z_{q,v})` in each metric the root allows (the target's at an operator root, none for @@ -501,7 +503,9 @@ execution `v` answering query `q`: a δ, one failure row per query, over every output `v` it may be answered from, by the bound `AccuracyTarget::failure_probability_bound` names from `V` (the values the target's scope covers) and the query's counts `R` and - fan-in (Pass 2's `ReadoutValueCounts`): + fan-in (Pass 2's `ReadoutValueCounts`). Pass 2 records the counts it split + δ by (`LogicalASAPMemoCandidates::readout_values_of_query`), and (K) + checks the bound they name: - every estimate read, when `R ≤ V · fan_in` or `V` is unbound: `W_v ≤ δ_q + M(1 − z_{q,v})`; - each covered value, when `V · fan_in < R` or `R` is unbound: @@ -527,7 +531,12 @@ estimates an output value depends on is decided by the data's series and keys (labels, group keys, join keys), not by the estimates. The exceptions are operators that choose by estimated value: a top-k readout is charged for every value it ranks, and any other such operator (a comparison, a `LIMIT` -after ordering by an estimate) has no contract. So for any data `D` the +after ordering by an estimate) has no contract. An operator keeping or +taking values by the keys a top-k chose (`p and topk(3, q)`, `p unless +topk(3, q)`, `topk(3, q) or p`, also through operators reading the top-k +value by value) depends on every estimate the top-k ranks: that input's +estimates count even where `β = 0`, and `or` sums its sides' `P` +(`asap_accuracy::values_chosen_by_estimate`). So for any data `D` the statement fails only on `⋃_{e ∈ S_q(D)} E_e`, where `S_q(D)` is the estimates the covered output values depend on and `E_e` is estimate `e`'s miss. Hence @@ -552,15 +561,19 @@ The recursions are sound step by step: An exact input has δ = 0. An input with `β = 0` contributes only which series or keys it has. A readout returns exactly the series the exact evaluation -does, so those estimates cannot make the output miss. +does, so those estimates cannot make the output miss, except a top-k's, +whose keys are chosen by its estimates (above). *The assumption it rests on.* Soundness needs `R_c` to be an upper bound on the estimates one evaluation of readout `c` computes, under every dataset the declared facts allow; it is the only declared count the bound trusts (`n_q` only decides which row is checked). `R_c` is -`Memo::max_values_per_evaluation`: the product of the distinct values its -state's keys' scans declare, at most the series for PromQL labels, clamped -to the rows its input reads. Two cases checked against it: +`Memo::max_estimates_per_evaluation`: for a readout other than a top-k its +values, `Memo::max_values_per_evaluation`, the product of the distinct +values its state's keys' scans declare, at most the series for PromQL +labels, clamped to the rows its input reads. The declared distinct items +are the deployment's contract, as HLL's are: declaring fewer than a state +counts gives a too small `R_c`. Two cases checked against it: - A top-k readout ranks every distinct item of each state, more than the values it returns. Its `R_c` is the smaller of its states' count times the declared distinct items per state (`SourceFacts::max_distinct_per_evaluation`) @@ -581,18 +594,18 @@ alternative. *Operator classes* (shared with Pass 2's split: `asap_accuracy::inputs_read_value_by_value`, -`takes_each_value_from_one_input` and the "error reaches the output" test -`ErrorMetricsOfMemo::input_error_reaches_output`): +`ErrorMetricsOfMemo::each_value_depends_on_one_input` and the "estimates +reach the output" test `ErrorMetricsOfMemo::input_estimates_reach_output`): | Class | Operators | `P_c` | `W_c` | |---|---|---|---| | Readout | `SummaryEstimate` (not top-k) | `δ_c` | `R_c δ_c` | | Readout choosing by estimate | top-k | `R_c δ_c` | `R_c δ_c` | | Value by value | filter on copied columns, project, sort, time shift, relabel, label enrichment, series sample; binary arithmetic, one-to-one or many-to-one (`on`, `ignoring`, `group_left`, `group_right`: each output value reads one value of each side, and the "one" side's value is read by many outputs but is one estimate); join | `Σ_i HP` | `Σ_i HW` | -| Each value from one input | `or`, concatenation (`UNION ALL`); `and`, `unless` (left only, the right has `β = 0`) | `max_i HP` | `Σ_i HW` | +| Each value from one input | `or`, concatenation (`UNION ALL`); `and`, `unless` (left only, the right has `β = 0`) | `max_i HP`; `Σ_i HP` when a top-k chooses an input's values | `Σ_i HW` | | Input read whole | aggregates (`sum`/`max`/`avg` by or without keys), any operator whose output value may read many values of an input, the operand of `scalar` | `Σ_i HW` | `Σ_i HW` | | No contract | a subquery, a comparison, a counter rate | none: no error, and so no estimate, passes | none | -| `β = 0` input | a count's input, the right of `and` and `unless` | adds 0 | adds 0 | +| `β = 0` input | a count's input, the right of `and` and `unless` | adds 0, or as read value by value or whole when a top-k chooses its values | adds 0, or `HW` when a top-k chooses its values | *Same rule as Pass 2, for a query planned alone.* Pass 2 gives each readout of a query one level `δ / min(R, V · fan_in)`. Its `R` and fan-in add, per @@ -663,13 +676,15 @@ target. The stricter tolerance is paid for only when the check needs it: solving every (K) at 1e-10 made RangeHeavy-8users about 6× slower, deriving the tolerance from the smallest limit solved RollupHeavy-8users about 2.8× slower, and scaling (K)'s rows by their limits solved RangeHeavy-8users -about 4× slower. +about 4× slower, and dividing every failure probability by the smallest +root limit (so the tightest limit is 1) solved RangeHeavy-8users 10% and +RangeHeavy-8users-counter 48% slower. **Context copies.** A copy of an equivalent evaluation context shares its original's variables and has no rows of its own. Two contexts are copies only when each output's root limits (the tolerated error per metric, and the -failure probability bound with its limit and, for each covered value, its -`V`) are equal, so the copy's rows would repeat the original's: +failure probability bound's kind and limit, all the rows read of it) are +equal, so the copy's rows would repeat the original's: - queries at δ = 0.01 per value returning 10 and 100 values are copies; - over a whole result they are copies when both check every estimate read (a readout of 10 series declaring 10 and 50 values);