diff --git a/Cargo.lock b/Cargo.lock index 0117bfb7b..703605ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,7 @@ version = "0.1.0" dependencies = [ "asap-types", "asap_sketchlib", + "serde", "serde_json", ] diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 666477f7e..7d2da01b9 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } +serde = { version = "1", features = ["derive"] } serde_json = "1" [dev-dependencies] diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 3b049a6f9..075ac794c 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -2,7 +2,8 @@ //! theoretical guarantee of reading a statistic from a summary with given //! parameters ([`theoretical_guarantee`], from the per-algorithm //! guarantees in `guarantee`), the deployment's measured accuracy -//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract +//! ([`AccuracyEvidence`], in `evidence`, and the measured table +//! [`MeasuredSketchAccuracy`], in `measured`), each operator's error contract //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the //! metrics each query root allows, [`query_roots_with_target`], and the @@ -20,6 +21,7 @@ mod error_contract; mod evidence; mod guarantee; mod logical_edge; +mod measured; mod metric_rule; mod readout; mod readout_caps; @@ -28,5 +30,9 @@ pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::{tolerated_limit, within_limit}; +pub use measured::{ + ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy, + MeasuredSketchAccuracyError, +}; pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/measured.rs b/crates/accuracy/src/measured.rs new file mode 100644 index 000000000..673b53b50 --- /dev/null +++ b/crates/accuracy/src/measured.rs @@ -0,0 +1,241 @@ +//! The deployment's measured sketch accuracy: the errors a calibration run +//! measured on the executor's own sketches, which Pass 2 reads through +//! [`AccuracyEvidence`] before each summary's theoretical guarantee. +//! +//! Example: KLL with `k = 800`, read for `Quantile { q: 0.99 }`, measured +//! over 1,000 trials, has rank error 0.00176 at failure probability 0.01 +//! and 0.00121 at 0.05. A request at failure probability 0.02 gets +//! 0.00176 (the largest recorded failure probability at most 0.02); a +//! request at 0.001 gets no measured guarantee, so the theoretical one +//! applies. + +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr}; +use asap_types::ir::schema::FieldDataType; +use asap_types::stages::ValueSign; +use asap_types::summary_type::{SketchParams, SketchStatistic}; +use serde::{Deserialize, Serialize}; + +use crate::guarantee::guarantee_before_failure_probability_check; +use crate::readout::readout_error_metric; +use crate::AccuracyEvidence; + +/// Errors measured on the executor's own sketches. Default: empty, every +/// sketch keeps its theoretical guarantee. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredSketchAccuracy { + pub entries: Vec, +} + +/// One sketch configuration, read for one statistic, in one metric. Each +/// recorded error is the worst case over the merge counts the calibration +/// measured: a state built over one window and a state merged from many +/// windows of the same θ share the entry, so its error must hold for +/// every one of them. Example: KLL k = 547 measured unmerged and merged +/// from 5 and 60 windows, with rank errors 0.0021, 0.0024 and 0.0023 at +/// δ = 0.01, records 0.0024 at 0.01. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredAccuracyEntry { + /// The exact parameters measured; only these match. + pub sketch: SketchParams, + /// The readout, e.g. `Quantile { q: 0.99 }`; only the same statistic + /// matches. + pub statistic: SketchStatistic, + /// The metric every recorded error is in, e.g. `Rank` for KLL. + pub accuracy_metric: AccuracyMetric, + /// Independent trials; every recorded failure probability is at least + /// 3 / trials. + pub trials: u32, + /// Ascending failure probability; the error does not increase as the + /// failure probability grows. + pub error_at: Vec, +} + +/// One measured point: the error not exceeded except with probability +/// `failure_probability`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErrorAtFailureProbability { + pub failure_probability: f64, + /// The (1 − δ) quantile of the per-value errors, the worst over the + /// measured merge counts, in the entry's `accuracy_metric`. + pub error: f64, +} + +/// Why a measured accuracy table is not valid. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum MeasuredSketchAccuracyError { + /// Entry `entry` has a failure probability outside (0, 1] or below + /// 3 / trials. + FailureProbabilityOutOfRange { + entry: usize, + failure_probability: f64, + }, + /// Entry `entry` records no point, so it can answer no request. + NoPoints { entry: usize }, + /// Entry `entry` has a negative or non-finite error. + InvalidError { entry: usize, error: f64 }, + /// Entry `entry`'s failure probabilities are not ascending, or its + /// error increases as the failure probability grows. + NotMonotone { entry: usize }, + /// Entries `first` and `second` measure the same sketch, statistic and + /// metric. + DuplicateEntry { first: usize, second: usize }, +} + +impl MeasuredSketchAccuracy { + /// Check every entry: at least one point, failure probabilities in + /// (0, 1] and at least 3 / trials, errors finite and non-negative, ascending failure + /// probability with non-increasing error, and no two entries for one + /// (sketch, statistic, metric). + pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> { + for (index, entry) in self.entries.iter().enumerate() { + entry.validate(index)?; + if let Some(first) = self.entries[..index].iter().position(|earlier| { + earlier.measures(&entry.sketch, &entry.statistic, entry.accuracy_metric) + }) { + return Err(MeasuredSketchAccuracyError::DuplicateEntry { + first, + second: index, + }); + } + } + Ok(()) + } + + /// The entry measuring `sketch` read for `statistic` in `metric`. + fn entry( + &self, + sketch: &SketchParams, + statistic: &SketchStatistic, + metric: AccuracyMetric, + ) -> Option<&MeasuredAccuracyEntry> { + self.entries + .iter() + .find(|entry| entry.measures(sketch, statistic, metric)) + } +} + +impl MeasuredAccuracyEntry { + /// Check this entry, the `index`th of its table. + fn validate(&self, index: usize) -> Result<(), MeasuredSketchAccuracyError> { + // Fewer than 3 failures in `trials` cannot estimate the (1 − δ) + // quantile: 1,000 trials record δ ≥ 0.003. + let smallest_failure_probability = 3.0 / f64::from(self.trials); + if self.error_at.is_empty() { + return Err(MeasuredSketchAccuracyError::NoPoints { entry: index }); + } + for point in &self.error_at { + let failure_probability = point.failure_probability; + if !(failure_probability > 0.0 + && failure_probability <= 1.0 + && failure_probability >= smallest_failure_probability) + { + return Err(MeasuredSketchAccuracyError::FailureProbabilityOutOfRange { + entry: index, + failure_probability, + }); + } + if !(point.error.is_finite() && point.error >= 0.0) { + return Err(MeasuredSketchAccuracyError::InvalidError { + entry: index, + error: point.error, + }); + } + } + let monotone = self.error_at.windows(2).all(|pair| { + pair[0].failure_probability < pair[1].failure_probability + && pair[0].error >= pair[1].error + }); + match monotone { + true => Ok(()), + false => Err(MeasuredSketchAccuracyError::NotMonotone { entry: index }), + } + } + + /// Whether this entry measures `sketch` read for `statistic` in + /// `metric`: the one key a table holds at most one entry for. + fn measures( + &self, + sketch: &SketchParams, + statistic: &SketchStatistic, + metric: AccuracyMetric, + ) -> bool { + self.sketch == *sketch && self.statistic == *statistic && self.accuracy_metric == metric + } + + /// The point at the largest recorded failure probability at most + /// `failure_probability`; `None` below the smallest (never + /// extrapolated). Example: recorded at 0.003, 0.01 and 0.05, a request + /// at 0.02 gets the point at 0.01. + fn point_at_most(&self, failure_probability: f64) -> Option { + self.error_at + .iter() + .filter(|point| point.failure_probability <= failure_probability) + .max_by(|a, b| a.failure_probability.total_cmp(&b.failure_probability)) + .copied() + } +} + +/// Whether θ's theoretical guarantee for `statistic` can be deterministic +/// (failure probability zero) under the data it requires: then a +/// measurement never replaces it. Example: DDSketch's relative error α +/// over values of one sign; a KLL's rank error always has a failure +/// probability. +fn has_deterministic_guarantee( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, +) -> bool { + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + 1.0, + Some(ValueSign::NonNegative), + None, + ) + .is_some_and(|guarantee| guarantee.spec.failure_probability == ProbabilityExpr::Zero) +} + +/// Loaders must call [`MeasuredSketchAccuracy::validate`] before handing a +/// table to the planner: `measured_guarantee` does not re-check it, so an +/// unvalidated table could give Pass 2 non-monotone or empty entries. +impl AccuracyEvidence for MeasuredSketchAccuracy { + /// The entry for exactly `parameters` and `statistic`, in the + /// readout's own metric (`Rank` for a KLL quantile), at its largest + /// recorded failure probability at most `failure_probability`; the + /// guarantee states that recorded failure probability. `None` when no + /// entry matches, below its smallest recorded failure probability, at + /// a failure probability that is not positive, and for a θ whose + /// theoretical guarantee can be deterministic (DDSketch). + fn measured_guarantee( + &self, + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, + ) -> Option { + if failure_probability.is_nan() || failure_probability <= 0.0 { + return None; + } + let metric = readout_error_metric(summary_type, statistic)?; + // Look the entry up first: the deterministic-guarantee check below + // computes a theoretical guarantee, so it runs only for a θ that + // has an entry. + let entry = self.entry(parameters, statistic, metric)?; + if has_deterministic_guarantee(summary_type, parameters, statistic) { + return None; + } + let point = entry.point_at_most(failure_probability)?; + Some(AccuracyGuarantee::new( + metric, + BoundExpr::Constant { value: point.error }, + ProbabilityExpr::Constant { + value: point.failure_probability, + }, + )) + } +} diff --git a/crates/accuracy/tests/measured_sketch_accuracy.rs b/crates/accuracy/tests/measured_sketch_accuracy.rs new file mode 100644 index 000000000..fb0632d33 --- /dev/null +++ b/crates/accuracy/tests/measured_sketch_accuracy.rs @@ -0,0 +1,383 @@ +//! The measured accuracy table as Pass 2's [`AccuracyEvidence`]: a request +//! at failure probability δ takes the largest recorded δ' ≤ δ of an entry +//! with exactly its parameters, statistic and metric, and nothing else. +//! Every expected value is read off the fixture by hand. + +use asap_accuracy::{ + theoretical_guarantee, AccuracyEvidence, ErrorAtFailureProbability, MeasuredAccuracyEntry, + MeasuredSketchAccuracy, MeasuredSketchAccuracyError, +}; +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; + +fn kll() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +fn p99() -> SketchStatistic { + SketchStatistic::Quantile { q: 0.99 } +} + +fn point(failure_probability: f64, error: f64) -> ErrorAtFailureProbability { + ErrorAtFailureProbability { + failure_probability, + error, + } +} + +/// KLL k = 547 read for the 0.99 quantile, rank error over 1,000 trials: +/// 0.0031 at δ = 0.003, 0.0024 at 0.01, 0.0016 at 0.05. +fn kll_547_entry() -> MeasuredAccuracyEntry { + MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k: 547 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Rank, + trials: 1_000, + error_at: vec![ + point(0.003, 0.0031), + point(0.01, 0.0024), + point(0.05, 0.0016), + ], + } +} + +fn table() -> MeasuredSketchAccuracy { + MeasuredSketchAccuracy { + entries: vec![kll_547_entry()], + } +} + +/// The error, failure probability and metric of a guarantee. +fn parts(guarantee: &AccuracyGuarantee) -> (f64, f64, AccuracyMetric) { + ( + guarantee.spec.error.evaluate().unwrap(), + guarantee.spec.failure_probability.evaluate().unwrap(), + guarantee.spec.accuracy_metric, + ) +} + +/// δ = 0.01 takes 0.01's error; 0.02 takes 0.01's too (the largest +/// recorded δ' ≤ 0.02, not the nearest 0.05 above); 0.05 and 1.0 (no +/// requirement) take 0.05's. +#[test] +fn lookup_takes_largest_recorded_delta_at_most_request() { + let table = table(); + let at = |delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + }; + assert_eq!(at(0.01), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.02), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.05), (0.0016, 0.05, AccuracyMetric::Rank)); + assert_eq!(at(1.0), (0.0016, 0.05, AccuracyMetric::Rank)); +} + +/// δ = 0.002 is below every recorded δ': no measured guarantee, never +/// 0.0031 extrapolated downward. +#[test] +fn lookup_no_extrapolation_below_smallest() { + let guarantee = + table().measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002); + assert_eq!(guarantee, None); +} + +/// Where the table has no measurement (δ = 0.002 above), the theoretical +/// guarantee applies: KLL k = 547 has rank error 2.296 / 547^0.9723 = +/// 0.004998 at δ ≥ 0.01, scaled below 0.01 by z(1 − δ/2) / z(0.995): +/// 0.004998 · 3.0902 / 2.5758 = 0.005997 at δ = 0.002. +#[test] +fn closed_form_applies_below_the_measured_range() { + let guarantee = theoretical_guarantee( + &kll(), + &SketchParams::Kll { k: 547 }, + &p99(), + 0.002, + None, + None, + ) + .unwrap(); + let (error, _, metric) = parts(&guarantee); + assert_eq!(metric, AccuracyMetric::Rank); + assert!((error - 0.005997).abs() < 5e-6, "{error}"); +} + +/// Only the exact parameters, statistic and metric match: k = 548, the +/// median, or the relative metric get nothing. +#[test] +fn lookup_exact_params_statistic_metric() { + let table = table(); + assert_eq!( + table.measured_guarantee(&kll(), &SketchParams::Kll { k: 548 }, &p99(), 0.01), + None + ); + assert_eq!( + table.measured_guarantee( + &kll(), + &SketchParams::Kll { k: 547 }, + &SketchStatistic::Quantile { q: 0.5 }, + 0.01 + ), + None + ); + let relative = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + accuracy_metric: AccuracyMetric::Relative, + ..kll_547_entry() + }], + }; + // An entry in another metric than the readout's own (Rank for KLL) is + // never returned. + assert_eq!( + relative.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.01), + None + ); +} + +/// An empty table measures nothing. +#[test] +fn empty_table_measures_nothing() { + let empty = MeasuredSketchAccuracy::default(); + for delta in [0.001, 0.01, 1.0] { + assert_eq!( + empty.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta), + None + ); + } +} + +/// Lookup monotonicity (I10): a larger δ never gets a larger measured +/// error than a smaller one. +#[test] +fn lookup_error_does_not_grow_with_delta() { + let table = table(); + let deltas = [0.003, 0.005, 0.01, 0.02, 0.05, 0.5, 1.0]; + let errors: Vec = deltas + .iter() + .map(|&delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + .0 + }) + .collect(); + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{errors:?}" + ); +} + +/// A deterministic sketch keeps its deterministic guarantee: a measured +/// (probabilistic) entry for DDSketch α = 0.01 is not returned. The rule +/// is general: the table is never read for a θ whose theoretical guarantee +/// has failure probability zero; DDSketch's relative α is the one such +/// case today. +#[test] +fn measured_entry_never_replaces_a_deterministic_guarantee() { + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: SketchParams::DDSketch { alpha: 0.01 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.004)], + }], + }; + let guarantee = table.measured_guarantee( + &FieldDataType::Sketch(SketchAlgorithm::DDSketch), + &SketchParams::DDSketch { alpha: 0.01 }, + &p99(), + 0.01, + ); + assert_eq!(guarantee, None); +} + +/// Whether a validation error is the expected one. +type IsReason = fn(&MeasuredSketchAccuracyError) -> bool; + +/// Each invalid table is rejected, for its own reason: the error variant +/// is asserted, so a table refused for the wrong reason fails the test. +#[test] +fn table_validation() { + use MeasuredSketchAccuracyError as E; + assert_eq!(table().validate(), Ok(())); + let with = |error_at: Vec| MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + error_at, + ..kll_547_entry() + }], + }; + let rejected: [(&str, MeasuredSketchAccuracy, IsReason); 9] = [ + // δ = 0.002 < 3 / 1,000 trials. + ( + "delta below 3/trials", + with(vec![point(0.002, 0.004)]), + |e| { + *e == E::FailureProbabilityOutOfRange { + entry: 0, + failure_probability: 0.002, + } + }, + ), + // An entry with no points answers no request. + ("no points", with(vec![]), |e| { + *e == E::NoPoints { entry: 0 } + }), + // The error grows with δ. + ( + "error increasing in delta", + with(vec![point(0.01, 0.002), point(0.05, 0.003)]), + |e| *e == E::NotMonotone { entry: 0 }, + ), + // δ not ascending. + ( + "unsorted deltas", + with(vec![point(0.05, 0.0016), point(0.01, 0.0024)]), + |e| *e == E::NotMonotone { entry: 0 }, + ), + ("delta zero", with(vec![point(0.0, 0.01)]), |e| { + *e == E::FailureProbabilityOutOfRange { + entry: 0, + failure_probability: 0.0, + } + }), + ("delta above one", with(vec![point(1.5, 0.01)]), |e| { + *e == E::FailureProbabilityOutOfRange { + entry: 0, + failure_probability: 1.5, + } + }), + ("negative error", with(vec![point(0.01, -1.0)]), |e| { + *e == E::InvalidError { + entry: 0, + error: -1.0, + } + }), + // NaN != NaN, so match the variant and check the error is NaN. + ( + "NaN error", + with(vec![point(0.01, f64::NAN)]), + |e| matches!(e, E::InvalidError { entry: 0, error } if error.is_nan()), + ), + ( + "duplicate entry", + MeasuredSketchAccuracy { + entries: vec![kll_547_entry(), kll_547_entry()], + }, + |e| { + *e == E::DuplicateEntry { + first: 0, + second: 1, + } + }, + ), + ]; + for (name, table, expected) in rejected { + let error = table.validate().expect_err(name); + assert!(expected(&error), "{name}: wrong reason {error:?}"); + } +} + +/// The table's JSON is the issue's example format, and round-trips. +#[test] +fn accuracy_json_round_trips() { + let text = r#"{"entries": [{"sketch": {"Kll": {"k": 800}}, "statistic": {"Quantile": {"q": 0.99}}, + "accuracy_metric": "rank", "trials": 1000, "error_at": [ + {"failure_probability": 0.01, "error": 0.00176}, + {"failure_probability": 0.05, "error": 0.00121}]}]}"#; + let table: MeasuredSketchAccuracy = serde_json::from_str(text).unwrap(); + assert_eq!(table.entries[0].sketch, SketchParams::Kll { k: 800 }); + assert_eq!(table.entries[0].error_at[1], point(0.05, 0.00121)); + let written = serde_json::to_string(&table).unwrap(); + assert_eq!( + serde_json::from_str::(&written).unwrap(), + table + ); +} + +/// A request at failure probability 0, below 0, or NaN has no recorded +/// δ′ ≤ δ: nothing is returned, and nothing panics. +#[test] +fn a_request_at_zero_or_nan_failure_probability_gets_nothing() { + let table = table(); + for delta in [0.0, -0.1, f64::NAN] { + assert_eq!( + table.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta), + None, + "δ = {delta}" + ); + } +} + +/// The returned guarantee states the recorded δ′ (0.01), not the requested +/// δ (0.02), so Pass 2's check that a guarantee holds at the requested +/// failure probability (0.01 ≤ 0.02) holds by construction. +#[test] +fn the_returned_guarantee_states_the_recorded_failure_probability() { + let guarantee = table() + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.02) + .unwrap(); + assert_eq!(guarantee.spec.failure_probability.evaluate(), Some(0.01)); + assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Rank); +} + +/// HLL's theoretical guarantee (relative standard error 1.04 / √m) has an +/// unknown failure probability, so Pass 2 prunes an HLL readout under a +/// target. A measured entry supplies one: HLL precision 13 read for the +/// distinct count, relative error 0.02 at δ = 0.01 over 1,000 trials, gives +/// the guarantee (0.02, 0.01, Relative). Measured evidence comes first, and +/// the deterministic rule does not apply (the theoretical failure +/// probability is unknown, not zero). The recorded error is the worst over +/// the merge counts the calibration measured, so the guarantee also holds +/// for a state merged from that many windows. +#[test] +fn a_measured_entry_supplies_a_guarantee_where_theory_states_no_failure_probability() { + let hll = FieldDataType::Sketch(SketchAlgorithm::Hll); + let parameters = SketchParams::Hll { precision: 13 }; + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: parameters.clone(), + statistic: SketchStatistic::Cardinality, + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.02)], + }], + }; + assert_eq!(table.validate(), Ok(())); + let guarantee = table + .measured_guarantee(&hll, ¶meters, &SketchStatistic::Cardinality, 0.01) + .unwrap(); + assert_eq!(parts(&guarantee), (0.02, 0.01, AccuracyMetric::Relative)); +} + +/// A readout with no metric of its own (UnivMon's distinct count has no +/// listed guarantee) never gets a measured one: there is no readout metric +/// for the entry's to match. +#[test] +fn an_entry_for_a_readout_with_no_metric_is_never_returned() { + let univmon = FieldDataType::Sketch(SketchAlgorithm::UnivMon); + let parameters = SketchParams::UnivMon { + heap_size: 32, + sketch_rows: 5, + sketch_cols: 1024, + layers: 8, + }; + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: parameters.clone(), + statistic: SketchStatistic::Cardinality, + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.05)], + }], + }; + assert_eq!( + table.measured_guarantee(&univmon, ¶meters, &SketchStatistic::Cardinality, 0.01), + None + ); +} diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 0d60737b0..fa3e46a54 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -31,8 +31,7 @@ mod resources; pub(crate) use measured::aggregate_lookup; pub use measured::{ - lookup, CostTableProvenance, MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, - OperatorCost, OperatorKind, StateFamily, SummaryOperation, + lookup, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, StateFamily, }; pub use model::{ CalibrationFit, ClusterLimits, ClusterObjective, CostInputs, CostModel, ResourceSpeed, diff --git a/crates/cost/src/measured.rs b/crates/cost/src/measured.rs index 7a660ce44..6e6c22c1b 100644 --- a/crates/cost/src/measured.rs +++ b/crates/cost/src/measured.rs @@ -33,10 +33,6 @@ //! (operations × `ms_per_cpu_op`) is the measured time, and every cost model //! prices it as it prices any operation. //! -//! The table of earlier versions, a JSON array of [`MeasuredOperationCost`] -//! (one sketch operation each), still loads: each entry becomes the -//! [`OperatorCost`] of its operator at both phases. -//! //! [`CostInputs::measured_operation_costs`]: crate::CostInputs::measured_operation_costs //! [`ResourceUsage`]: crate::ResourceUsage //! [`ResourceSpeed`]: crate::ResourceSpeed @@ -53,32 +49,19 @@ use crate::node::analytical::AnalyticalCostError; pub use asap_types::ir::operator::OperatorKind; /// The deployment's measured cost table -/// ([`CostInputs::measured_operation_costs`]). In JSON, an object -/// with these fields, or (the format of earlier versions) an array of -/// [`MeasuredOperationCost`]. Default: empty, every node priced from the -/// formulas. +/// ([`CostInputs::measured_operation_costs`]). In JSON, an object with +/// these fields; an omitted field takes its default, an unknown one is +/// refused. Default: empty, every node priced from the formulas. /// /// [`CostInputs::measured_operation_costs`]: crate::CostInputs::measured_operation_costs -#[derive(Debug, Clone, Default, PartialEq, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] pub struct MeasuredOperationCosts { /// One entry per (operator, phase, family) the runtime measured. pub operators: Vec, /// CPU nanoseconds the runtime pays once per query request, whatever /// its plan: scheduling, request handling and result encoding. Default 0. pub ns_per_evaluation: f64, - /// Which executor and machine the numbers were measured with; the table - /// is re-measured when either changes. Recorded, not read. - pub provenance: Option, -} - -/// Where a measured table's numbers come from. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CostTableProvenance { - /// The executor build measured, e.g. a crate version or commit. - pub executor_version: String, - /// The machine it ran on, e.g. a CPU model or instance type. - pub machine: String, } /// The measured CPU time of one operator kind, at one phase, for one family. @@ -337,99 +320,6 @@ pub(crate) fn validate( Ok(()) } -// ── The format of earlier versions ────────────────────────────────────── - -/// One operation on a sketch's state, as an entry of the earlier format -/// ([`MeasuredOperationCost`]) names it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryOperation { - /// Insert one input row: a `SummaryAgg` or `SummaryTumblingWindowChain` entry. - Update, - /// Merge one input state into another: a `SummaryMerge` entry. - Merge, - /// Read one result out of one state: a `SummaryEstimate` entry. - Estimate, -} - -/// An entry of the earlier format, still loadable: the measured CPU time of -/// one operation on one sketch configuration. It becomes the -/// [`OperatorCost`] of its operator (both `SummaryAgg` and `SummaryTumblingWindowChain` for -/// an update) at both phases. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MeasuredOperationCost { - /// The sketch and its parameters, e.g. `{"Kll": {"k": 200}}`. - pub sketch: SketchParams, - /// Which operation was measured. - pub operation: SummaryOperation, - /// CPU nanoseconds one such operation takes on one core. - pub cpu_ns_per_operation: f64, -} - -impl From> for MeasuredOperationCosts { - fn from(entries: Vec) -> Self { - let operators = entries - .into_iter() - .flat_map(|entry| { - let kinds: &[OperatorKind] = match entry.operation { - SummaryOperation::Update => &[ - OperatorKind::SummaryAgg, - OperatorKind::SummaryTumblingWindowChain, - ], - SummaryOperation::Merge => &[OperatorKind::SummaryMerge], - SummaryOperation::Estimate => &[OperatorKind::SummaryEstimate], - }; - kinds - .iter() - .map(|&operator| OperatorCost { - operator, - phase: None, - sketch: Some(entry.sketch.clone()), - exact: None, - measure: None, - ns_per_unit: entry.cpu_ns_per_operation, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - }) - .collect::>() - }) - .collect(); - Self { - operators, - ..Self::default() - } - } -} - -/// Reads the table, or an array in the earlier format. -impl<'de> Deserialize<'de> for MeasuredOperationCosts { - fn deserialize>(deserializer: D) -> Result { - use serde::de::Error; - let value = serde_json::Value::deserialize(deserializer)?; - if value.is_array() { - return Vec::::deserialize(value) - .map(Self::from) - .map_err(D::Error::custom); - } - let fields = TableFields::deserialize(value).map_err(D::Error::custom)?; - Ok(Self { - operators: fields.operators, - ns_per_evaluation: fields.ns_per_evaluation, - provenance: fields.provenance, - }) - } -} - -/// The table's fields as JSON states them; omitted ones take their default. -#[derive(Default, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct TableFields { - operators: Vec, - ns_per_evaluation: f64, - provenance: Option, -} - /// Which costs one node's CPU operations came from. A node is one operator /// node of the candidate's physical plan DAG (`PhysicalASAPDAG`), keyed by /// its `PhysicalASAPNodeId` in [`PlanCost::per_node`]; not a machine. diff --git a/crates/cost/src/model.rs b/crates/cost/src/model.rs index 40e16c35c..9908f6a5c 100644 --- a/crates/cost/src/model.rs +++ b/crates/cost/src/model.rs @@ -240,7 +240,7 @@ pub struct CostInputs { /// The deployment's measured CPU time per unit of each operator; a node /// with an entry is priced from it, every other node from the formulas /// scaled by the calibration factor (module doc, "Measured operation - /// costs"). Also loads the earlier array of [`MeasuredOperationCost`](crate::MeasuredOperationCost). + /// costs"). /// Default empty: every node is priced from the formulas. pub measured_operation_costs: MeasuredOperationCosts, /// Externally established formula-to-measurement scale, held constant across diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 407ddb326..611aaf6be 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -3364,3 +3364,148 @@ fn group_visit_units_follow_the_raw_batch_width() { assert_eq!(visits(1_000), 300_000.0); assert_eq!(visits(10_000), 30_000.0); } + +/// A query-time producer of `rows` plain rows, for a node under test. +fn rows_of<'a>(producer: &'a PhysicalASAPDAGNode, rows: u64) -> ProducerOutput<'a> { + ProducerOutput { + payload: &producer.payload, + output_schema: &producer.output_schema, + output: EdgeStatistics { + rows, + bytes: rows * 16, + }, + sources: BTreeSet::new(), + family: StateFamily::None, + unchanged_columns: BTreeSet::new(), + tumbling_windows: 1, + chain_tumbling_window_width_ms: None, + state_input_rows: None, + raw_rows_span_ms: None, + } +} + +/// A one-minute `TimeRange` over node 0. +fn one_minute_range() -> PhysicalASAPDAGNode { + node(Payload::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: 0, + })) +} + +/// A query-time `SummaryAgg` of an exact sum over 600 input rows, with a +/// measured entry of 10 ns per row and 500 ns per computation, takes +/// 600 · 10 + 500 = 6,500 ns = 0.0065 ms per execution; computed once a +/// second, 6.5e-6 core-seconds per second, from the table. +#[test] +fn node_priced_by_hand() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::SummaryAgg, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 10.0, + ns_per_invocation: 500.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let build = node(Payload::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })); + let usage = node_resource_usage_and_cost( + &build, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.0065).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); + assert!( + (usage.cost.cpu_core_seconds_per_second - 6.5e-6).abs() < 1e-15, + "{}", + usage.cost.cpu_core_seconds_per_second + ); +} + +/// An `Aggregate` is priced from the table only when each of its measures +/// has an entry: with an entry for `sum` (10 ns per row) and none for +/// `quantile`, `Aggregate[sum, quantile]` keeps the formulas, and +/// `Aggregate[sum]` over 600 rows takes 600 · 10 = 6,000 ns. +#[test] +fn aggregate_needs_every_measure() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("sum".into()), + ns_per_unit: 10.0, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let aggregate = |measures: Vec| { + node(Payload::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::none()), + output_names: vec![String::new(); measures.len()], + filters: vec![None; measures.len()], + measures, + having: None, + child: 0, + })) + }; + let both = aggregate(vec![ + AggIntent::Sum { col: None }, + AggIntent::Quantile { col: None, q: 0.5 }, + ]); + let usage = node_resource_usage_and_cost( + &both, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Analytical); + let sum = aggregate(vec![AggIntent::Sum { col: None }]); + let usage = node_resource_usage_and_cost( + &sum, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.006).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); +} diff --git a/crates/cost/tests/measured_operation_costs.rs b/crates/cost/tests/measured_operation_costs.rs new file mode 100644 index 000000000..047fe473d --- /dev/null +++ b/crates/cost/tests/measured_operation_costs.rs @@ -0,0 +1,260 @@ +//! The measured cost table: [`lookup`] (exact match, interpolation in log +//! size, phase shadowing, no extrapolation), its JSON round trip and the +//! formats the `asap_run` design deletes. Every expected price is +//! computed by hand from the entries below. + +use asap_cost::{ + lookup, CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind, StateFamily, +}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::summary_type::{ExactKind, SketchParams}; + +/// A sketch's size for interpolation: KLL's `k`, 1 for any other sketch. +fn size_of(params: &SketchParams) -> f64 { + match params { + SketchParams::Kll { k } => *k as f64, + _ => 1.0, + } +} + +/// A `SummaryAgg` entry for KLL with `k` at `phase`. +fn kll_entry( + k: u32, + phase: Option, + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +) -> OperatorCost { + OperatorCost { + operator: OperatorKind::SummaryAgg, + phase, + sketch: Some(SketchParams::Kll { k }), + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation, + memory_bytes_per_state, + } +} + +/// KLL `k = 200` at 100 ns/unit, 1,000 ns/invocation, 2,000 bytes, and +/// `k = 800` at 300 ns/unit, 3,000 ns/invocation, 8,000 bytes, for both +/// phases. +fn kll_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, Some(8_000.0)), + ], + ..MeasuredOperationCosts::default() + } +} + +/// A price's fields: ns per unit, ns per invocation, bytes per state. +#[derive(Debug, PartialEq)] +struct Price { + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +} + +fn kll_price_at(table: &MeasuredOperationCosts, phase: ExecutionTiming, k: u32) -> Option { + lookup( + table, + OperatorKind::SummaryAgg, + phase, + StateFamily::Sketch(&SketchParams::Kll { k }), + size_of, + ) + .map(|price| Price { + ns_per_unit: price.ns_per_unit, + ns_per_invocation: price.ns_per_invocation, + memory_bytes_per_state: price.memory_bytes_per_state, + }) +} + +fn assert_close(actual: f64, expected: f64) { + assert!( + (actual - expected).abs() <= 1e-9 * expected.abs().max(1.0), + "{actual} != {expected}" + ); +} + +/// An entry with exactly the node's parameters is its price, not an +/// interpolation. +#[test] +fn lookup_exact_params_win() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 200).unwrap(); + assert_eq!(price.ns_per_unit, 100.0); + assert_eq!(price.ns_per_invocation, 1_000.0); + assert_eq!(price.memory_bytes_per_state, Some(2_000.0)); +} + +/// k = 400 lies at ln(400/200) / ln(800/200) = 0.5 between the two +/// entries in log size: 100 + 0.5·200 = 200 ns/unit, 2,000 ns/invocation, +/// 5,000 bytes. Linear in size would give 166.7 ns/unit. +#[test] +fn lookup_interpolates_in_log_size() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 400).unwrap(); + assert_close(price.ns_per_unit, 200.0); + assert_close(price.ns_per_invocation, 2_000.0); + assert_close(price.memory_bytes_per_state.unwrap(), 5_000.0); +} + +/// Below the smallest and above the largest measured size, the nearest +/// entry's price: 100 at k = 100 (not 33), 300 at k = 1600 (not 367). +#[test] +fn lookup_no_extrapolation() { + let table = kll_table(); + let below = kll_price_at(&table, ExecutionTiming::QueryTime, 100).unwrap(); + let above = kll_price_at(&table, ExecutionTiming::QueryTime, 1_600).unwrap(); + assert_eq!(below.ns_per_unit, 100.0); + assert_eq!(above.ns_per_unit, 300.0); +} + +/// A query-time entry for k = 200 at 999 ns/unit hides both both-phase +/// entries at query time: k = 400 has only k = 200 to go by, so 999. At +/// ingestion time there is no ingestion entry, so the both-phase pair +/// interpolates to 200. +#[test] +fn lookup_phase_entries_shadow_both() { + let mut table = kll_table(); + table.operators.push(kll_entry( + 200, + Some(ExecutionTiming::QueryTime), + 999.0, + 0.0, + None, + )); + let query = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + let ingestion = kll_price_at(&table, ExecutionTiming::IngestionTime, 400).unwrap(); + assert_eq!(query.ns_per_unit, 999.0); + assert_close(ingestion.ns_per_unit, 200.0); +} + +/// With no bytes on k = 800, k = 400 has one memory neighbour only: no +/// measured bytes (the formulas' bytes apply), never an interpolation +/// with 0. +#[test] +fn lookup_memory_needs_both_neighbours() { + let table = MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, None), + ], + ..MeasuredOperationCosts::default() + }; + let price = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + assert_eq!(price.memory_bytes_per_state, None); +} + +/// KLL entries do not price a DDSketch node. +#[test] +fn lookup_other_algorithm_misses() { + let price = lookup( + &kll_table(), + OperatorKind::SummaryAgg, + ExecutionTiming::QueryTime, + StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), + size_of, + ); + assert!(price.is_none()); +} + +/// A table with one of everything a JSON file can hold. +fn full_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry( + 200, + Some(ExecutionTiming::IngestionTime), + 100.5, + 1_000.25, + Some(2_048.0), + ), + OperatorCost { + operator: OperatorKind::SummaryMerge, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 3.75, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("quantile".into()), + ns_per_unit: 12.0, + ns_per_invocation: 250.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: 20.0, + ns_per_invocation: 5_000.0, + memory_bytes_per_state: None, + }, + ], + ns_per_evaluation: 50_000.0, + } +} + +/// `costs.json` round-trips: written, read back and written again it is +/// byte-equal, and it validates. +#[test] +fn costs_json_round_trips() { + let table = full_table(); + let written = serde_json::to_string_pretty(&table).unwrap(); + let read: MeasuredOperationCosts = serde_json::from_str(&written).unwrap(); + assert_eq!(read, table); + assert_eq!(serde_json::to_string_pretty(&read).unwrap(), written); + let inputs = CostInputs { + measured_operation_costs: read, + ..CostInputs::default() + }; + inputs.validate().unwrap(); +} + +/// `provenance` is deleted (where a table was measured is recorded by the +/// calibration run, not in the table) and so is the array format of +/// earlier versions: a table carrying either no longer loads. +#[test] +fn costs_json_rejects_removed_fields() { + let with_provenance = r#"{"operators": [], "ns_per_evaluation": 0.0, + "provenance": {"executor_version": "0.1.0", "machine": "x86_64"}}"#; + assert!( + serde_json::from_str::(with_provenance).is_err(), + "a table with `provenance` must not load" + ); + let legacy_array = r#"[{"sketch": {"Kll": {"k": 200}}, "operation": "update", + "cpu_ns_per_operation": 100.0}]"#; + assert!( + serde_json::from_str::(legacy_array).is_err(), + "a table in the array format must not load" + ); +} + +/// A `CostInputs` file (the `asap_planner --cost-inputs` format) whose +/// measured table still carries `provenance` is refused, and the message +/// names the field, so a stale file fails loudly instead of loading with +/// the field ignored. +#[test] +fn a_cost_inputs_file_with_provenance_is_refused() { + let mut inputs = serde_json::to_value(CostInputs::default()).unwrap(); + inputs["measured_operation_costs"] = serde_json::json!({ + "operators": [], + "ns_per_evaluation": 0.0, + "provenance": {"executor_version": "0.1.0", "machine": "x86_64"} + }); + let error = serde_json::from_value::(inputs) + .expect_err("a cost-inputs file with `provenance` must not load"); + assert!(error.to_string().contains("provenance"), "{error}"); +} diff --git a/crates/devtools/src/bin/convert_rqe_atomic_costs.rs b/crates/devtools/src/bin/convert_rqe_atomic_costs.rs index a08cdcca8..949b4d485 100644 --- a/crates/devtools/src/bin/convert_rqe_atomic_costs.rs +++ b/crates/devtools/src/bin/convert_rqe_atomic_costs.rs @@ -2,9 +2,8 @@ // optimizer_cost/rqe_atomic_costs.json measured_operation_costs.json // // Reads sketch-bench's per-operation cost table (`rqe_atomic_costs.json`) -// and writes plan selection's measured table: a JSON array of -// `{"sketch": , "operation": "update|merge|estimate", -// "cpu_ns_per_operation": }`, three entries per converted row +// and writes plan selection's measured table in its object form +// (`MeasuredOperationCosts`): four `OperatorCost` entries per converted row // (`asap_devtools::rqe_atomic_costs`). Set it as // `CostInputs::measured_operation_costs` (`asap_planner --cost-inputs`). Skipped rows (exact // accumulators, Hydra structures, unknown names) are listed on stderr. @@ -27,7 +26,7 @@ fn main() -> Result<(), Box> { )?; eprintln!( "wrote {} entries to {output}", - converted.measured_operation_costs.len() + converted.measured_operation_costs.operators.len() ); Ok(()) } diff --git a/crates/devtools/src/rqe_atomic_costs.rs b/crates/devtools/src/rqe_atomic_costs.rs index faccfed55..bd7679804 100644 --- a/crates/devtools/src/rqe_atomic_costs.rs +++ b/crates/devtools/src/rqe_atomic_costs.rs @@ -9,12 +9,16 @@ //! "insert_cpu_secs": 8.5e-8, "merge_cpu_secs": 1.2e-6, "query_cpu_secs": 4.0e-7, ...} //! ``` //! -//! and each becomes three entries, one per operation, in CPU nanoseconds: +//! and each becomes four `OperatorCost` entries of the table's object form, +//! in CPU nanoseconds per unit, for both phases: //! //! ```json -//! [{"sketch": {"Kll": {"k": 200}}, "operation": "update", "cpu_ns_per_operation": 85.0}, -//! {"sketch": {"Kll": {"k": 200}}, "operation": "merge", "cpu_ns_per_operation": 1200.0}, -//! {"sketch": {"Kll": {"k": 200}}, "operation": "estimate", "cpu_ns_per_operation": 400.0}] +//! {"operators": [ +//! {"operator": "SummaryAgg", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 85.0, "ns_per_invocation": 0.0}, +//! {"operator": "SummaryTumblingWindowChain", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 85.0, "ns_per_invocation": 0.0}, +//! {"operator": "SummaryMerge", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 1200.0, "ns_per_invocation": 0.0}, +//! {"operator": "SummaryEstimate", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 400.0, "ns_per_invocation": 0.0}], +//! "ns_per_evaluation": 0.0} //! ``` //! //! Only this converter reads the sketch-bench format; the planner reads the @@ -24,7 +28,7 @@ //! //! [`CostInputs::measured_operation_costs`]: asap_cost::CostInputs::measured_operation_costs -use asap_cost::{MeasuredOperationCost, SummaryOperation}; +use asap_cost::{MeasuredOperationCosts, OperatorCost, OperatorKind}; use asap_types::summary_type::SketchParams; use serde_json::Value; @@ -40,9 +44,11 @@ pub struct SkippedRow { /// What [`convert_rqe_atomic_costs`] produced. #[derive(Debug, Clone, PartialEq)] pub struct ConvertedCosts { - /// The measured table: three entries per converted row (update, merge, - /// estimate), in row order. - pub measured_operation_costs: Vec, + /// The measured table in its object form: per converted row, an update + /// becomes `SummaryAgg` and `SummaryTumblingWindowChain` entries, a merge a + /// `SummaryMerge` entry and an estimate a `SummaryEstimate` entry, each + /// for both phases, in row order. + pub measured_operation_costs: MeasuredOperationCosts, /// The rows skipped, in row order. pub skipped_rows: Vec, } @@ -55,10 +61,8 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result let rows = rows .as_array() .ok_or("rqe_atomic_costs.json is not a JSON array of rows")?; - let mut converted = ConvertedCosts { - measured_operation_costs: Vec::new(), - skipped_rows: Vec::new(), - }; + let mut operators: Vec = Vec::new(); + let mut skipped_rows = Vec::new(); for (index, row) in rows.iter().enumerate() { let sketch_name = row["sketch"] .as_str() @@ -67,7 +71,7 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result let sketch = match sketch_params(sketch_name, params) { Ok(Some(sketch)) => sketch, Ok(None) => { - converted.skipped_rows.push(SkippedRow { + skipped_rows.push(SkippedRow { sketch_name: sketch_name.to_string(), reason: skip_reason(sketch_name).to_string(), }); @@ -75,19 +79,24 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result } Err(error) => return Err(format!("row {index} ({sketch_name}): {error}")), }; - if converted - .measured_operation_costs + if operators .iter() - .any(|entry| entry.sketch == sketch) + .any(|entry| entry.sketch.as_ref() == Some(&sketch)) { return Err(format!( "row {index} ({sketch_name}): a second row for {sketch:?}" )); } - for (field, operation) in [ - ("insert_cpu_secs", SummaryOperation::Update), - ("merge_cpu_secs", SummaryOperation::Merge), - ("query_cpu_secs", SummaryOperation::Estimate), + // An update prices both operators that fold rows into a state; a + // merge and an estimate their one operator. + let updates: &[OperatorKind] = &[ + OperatorKind::SummaryAgg, + OperatorKind::SummaryTumblingWindowChain, + ]; + for (field, kinds) in [ + ("insert_cpu_secs", updates), + ("merge_cpu_secs", &[OperatorKind::SummaryMerge]), + ("query_cpu_secs", &[OperatorKind::SummaryEstimate]), ] { let seconds = row[field] .as_f64() @@ -95,16 +104,25 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result .ok_or(format!( "row {index} ({sketch_name}): `{field}` is not a non-negative number" ))?; - converted - .measured_operation_costs - .push(MeasuredOperationCost { - sketch: sketch.clone(), - operation, - cpu_ns_per_operation: seconds * 1e9, - }); + operators.extend(kinds.iter().map(|&operator| OperatorCost { + operator, + phase: None, + sketch: Some(sketch.clone()), + exact: None, + measure: None, + ns_per_unit: seconds * 1e9, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + })); } } - Ok(converted) + Ok(ConvertedCosts { + measured_operation_costs: MeasuredOperationCosts { + operators, + ..MeasuredOperationCosts::default() + }, + skipped_rows, + }) } /// Why a row whose name has no planner sketch is skipped. @@ -183,10 +201,14 @@ mod tests { /// Four sketch-bench rows in the format of `rqe_atomic_costs.json`: a /// KLL and a Count-Min top-k convert, the exact sum and the Hydra HLL - /// are skipped. The table written as JSON reads back as the same - /// entries, with seconds turned into nanoseconds. + /// are skipped. Each converted row gives four `OperatorCost` entries for + /// both phases (an update prices `SummaryAgg` and + /// `SummaryTumblingWindowChain`, a merge `SummaryMerge`, an estimate + /// `SummaryEstimate`), with seconds turned into nanoseconds per unit; + /// the table written as JSON reads back equal. #[test] fn a_small_sample_converts_and_round_trips() { + use asap_cost::{OperatorCost, OperatorKind}; let sample = json!([ {"sketch": "kll-percall", "sketch_config": {"algorithm": "kll-percall", "params": {"k": 200}}, "mem_bytes_per_instance": 9648.0, "insert_cpu_secs": 8.5e-8, "merge_cpu_secs": 1.2e-6, @@ -208,29 +230,44 @@ mod tests { depth: 3, heap_size: 32, }; - let expected: Vec<_> = [ - (&kll, SummaryOperation::Update, 85.0), - (&kll, SummaryOperation::Merge, 1200.0), - (&kll, SummaryOperation::Estimate, 400.0), - (&cms, SummaryOperation::Update, 60.0), - (&cms, SummaryOperation::Merge, 5000.0), - (&cms, SummaryOperation::Estimate, 350.0), - ] - .into_iter() - .map(|(sketch, operation, ns)| (sketch.clone(), operation, ns)) - .collect(); - let got: Vec<_> = converted - .measured_operation_costs - .iter() - .map(|e| (e.sketch.clone(), e.operation, e.cpu_ns_per_operation)) - .collect(); - assert_eq!(got.len(), expected.len()); - for ((sketch, operation, ns), (want_sketch, want_operation, want_ns)) in - got.iter().zip(&expected) - { - assert_eq!((sketch, operation), (want_sketch, want_operation)); - assert!((ns - want_ns).abs() < 1e-9 * want_ns, "{ns} vs {want_ns}"); + let expected = [ + (&kll, OperatorKind::SummaryAgg, 85.0), + (&kll, OperatorKind::SummaryTumblingWindowChain, 85.0), + (&kll, OperatorKind::SummaryMerge, 1200.0), + (&kll, OperatorKind::SummaryEstimate, 400.0), + (&cms, OperatorKind::SummaryAgg, 60.0), + (&cms, OperatorKind::SummaryTumblingWindowChain, 60.0), + (&cms, OperatorKind::SummaryMerge, 5000.0), + (&cms, OperatorKind::SummaryEstimate, 350.0), + ]; + let table = &converted.measured_operation_costs; + assert_eq!(table.operators.len(), expected.len()); + for (entry, (sketch, operator, ns)) in table.operators.iter().zip(expected) { + let OperatorCost { + operator: got_operator, + phase, + sketch: got_sketch, + exact, + measure, + ns_per_unit, + ns_per_invocation, + memory_bytes_per_state, + } = entry; + assert_eq!( + (got_operator, got_sketch.as_ref(), *phase), + (&operator, Some(sketch), None) + ); + assert_eq!( + (exact, measure, memory_bytes_per_state), + (&None, &None, &None) + ); + assert_eq!(*ns_per_invocation, 0.0); + assert!( + (ns_per_unit - ns).abs() < 1e-9 * ns, + "{ns_per_unit} vs {ns}" + ); } + assert_eq!(table.ns_per_evaluation, 0.0); let skipped: Vec<_> = converted .skipped_rows .iter() @@ -238,9 +275,9 @@ mod tests { .collect(); assert_eq!(skipped, ["exact-sum", "hydra-hll"]); - let written = serde_json::to_string(&converted.measured_operation_costs).unwrap(); - let read: Vec = serde_json::from_str(&written).unwrap(); - assert_eq!(read, converted.measured_operation_costs); + let written = serde_json::to_string(table).unwrap(); + let read: MeasuredOperationCosts = serde_json::from_str(&written).unwrap(); + assert_eq!(&read, table); } /// Bad input is an error, not a skipped row: a missing time, a negative diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index b7b0cf9ce..6002be097 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -60,9 +60,8 @@ pub use installable::{ // What a deployment runtime needs to build a `UserInput` and read an // `InstallablePlan`, so it depends on this crate and `asap-executor` alone. pub use asap_cost::{ - ClusterLimits, ClusterObjective, CostInputs, CostModel, CostTableProvenance, - MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, - ResourceSpeed, ResourceUnitCost, SummaryOperation, + ClusterLimits, ClusterObjective, CostInputs, CostModel, MeasuredOperationCosts, NodeCostSource, + OperatorCost, OperatorKind, ResourceSpeed, ResourceUnitCost, }; pub use asap_frontend_sql::SqlCatalog; pub use asap_stage3_plan_selection::{ diff --git a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs index f0e48388d..90e2293d9 100644 --- a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs @@ -5,7 +5,7 @@ //! its output. use std::collections::BTreeSet; -use asap_accuracy::NoMeasuredGuarantees; +use asap_accuracy::{AccuracyEvidence, NoMeasuredGuarantees}; use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; @@ -59,7 +59,17 @@ pub fn entry(interval_ms: u32) -> QueryWorkloadEntry { } pub fn pass2(memo: Memo, entries: &[QueryWorkloadEntry]) -> LogicalASAPMemoCandidates { - generate_logical_asap_memo_candidates(memo, entries, &NoMeasuredGuarantees).unwrap() + pass2_with_evidence(memo, entries, &NoMeasuredGuarantees) +} + +/// [`pass2`] with the deployment's measured accuracy `evidence`, read +/// before each summary's theoretical guarantee. +pub fn pass2_with_evidence( + memo: Memo, + entries: &[QueryWorkloadEntry], + evidence: &dyn AccuracyEvidence, +) -> LogicalASAPMemoCandidates { + generate_logical_asap_memo_candidates(memo, entries, evidence).unwrap() } /// The edges into input `input` of `consumer`: each producer and α^log. diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs index fe726caf0..2883c676b 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs @@ -1,7 +1,8 @@ //! Steps 4 to 6: the logical edges Pass 2 assembles and keeps ($\alpha^{\log}$ //! and the metric rule themselves are tested in `asap_accuracy`). -use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_accuracy::{ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy}; +use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ @@ -120,3 +121,151 @@ fn a_merge_is_judged_by_the_parameters_it_carries() { } if max_error == 0.005 ))); } + +/// A measured accuracy table with one entry: `sketch` read for the 0.99 +/// quantile in `accuracy_metric`, over 1,000 trials, with `error` at +/// failure probability 0.01. +fn measured_p99( + sketch: SketchParams, + accuracy_metric: AccuracyMetric, + error: f64, +) -> MeasuredSketchAccuracy { + MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric, + trials: 1_000, + error_at: vec![ErrorAtFailureProbability { + failure_probability: 0.01, + error, + }], + }], + } +} + +/// `quantile_over_time(0.99, m[5m])` over one series of `m` (so δ is not +/// split across readout values) whose values are non-negative (so +/// DDSketch's sign condition holds), with a per-value `metric` target of +/// `max_error` at δ = 0.01: the memo, its state group and readout group. +fn p99_with_per_value_target( + algorithm: SketchAlgorithm, + metric: AccuracyMetric, + max_error: f64, +) -> (Memo, MemoGroupId, MemoGroupId) { + let mut memo = Memo::new(&with_non_negative_values("m").with_series_count(1)); + let (_, state, readout) = quantile_over_time(&mut memo, algorithm, 0.99, 300); + query( + &mut memo, + readout, + Some( + AccuracyTarget::max_error(max_error) + .failure_probability(0.01) + .accuracy_metric(metric) + .accuracy_scope(AccuracyScope::PerValue), + ), + ); + (memo, state, readout) +} + +/// The build candidate of `state` with parameters `parameters`. +fn build_with( + logical: &crate::LogicalASAPMemoCandidates, + state: MemoGroupId, + parameters: &SketchParams, +) -> CandidateForMemoGroupId { + let index_in_group = candidates_of(&logical.memo, state) + .iter() + .position(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryAgg { .. }) + ) && c.logical_properties.parameters() == Some(parameters) + }) + .unwrap_or_else(|| panic!("a build with {parameters:?} in the state group")); + CandidateForMemoGroupId { + group: state, + index_in_group, + } +} + +/// Whether `candidate` is pruned, every cause `BelowRequiredAccuracy`. +fn pruned_below_required_accuracy( + logical: &crate::LogicalASAPMemoCandidates, + candidate: CandidateForMemoGroupId, +) -> bool { + pruned_entry(logical, candidate).is_some_and(|pruned| { + !pruned.causes.is_empty() + && pruned + .causes + .iter() + .all(|cause| matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. })) + }) +} + +/// Measured accuracy keeps a smaller KLL that theory prunes. Rank target +/// 0.0025 per value at δ = 0.01: in theory KLL k = 547 has rank error +/// 2.296 / 547^0.9723 = 0.004998 > 0.0025 and is pruned, k = 1,404 has +/// 0.00200 and is kept. A table measuring k = 547 at rank error 0.0024 +/// at δ′ = 0.01 (≤ 0.0025) keeps the k = 547 edge. +#[test] +fn measured_accuracy_keeps_a_smaller_kll_that_theory_prunes() { + let k547 = SketchParams::Kll { k: 547 }; + let k1404 = SketchParams::Kll { k: 1404 }; + + let (memo, state, readout) = + p99_with_per_value_target(SketchAlgorithm::Kll, AccuracyMetric::Rank, 0.0025); + let theory = pass2(memo, &[entry(60_000)]); + assert!( + pruned_below_required_accuracy(&theory, build_with(&theory, state, &k547)), + "without evidence, k = 547 is below the required accuracy" + ); + let into_readout = edges_into(&theory, own_candidate(readout), 0); + assert!( + into_readout.contains(&(build_with(&theory, state, &k1404), true)), + "{into_readout:?}" + ); + + let (memo, state, readout) = + p99_with_per_value_target(SketchAlgorithm::Kll, AccuracyMetric::Rank, 0.0025); + let measured = measured_p99(k547.clone(), AccuracyMetric::Rank, 0.0024); + let with_table = pass2_with_evidence(memo, &[entry(60_000)], &measured); + let build_547 = build_with(&with_table, state, &k547); + assert!(pruned_entry(&with_table, build_547).is_none()); + let into_readout = edges_into(&with_table, own_candidate(readout), 0); + assert!( + into_readout.contains(&(build_547, true)), + "{into_readout:?}" + ); +} + +/// Measured accuracy never rescues a DDSketch α: its relative guarantee is +/// deterministic (failure probability zero), so the table is not read. At +/// a per-value relative target of 0.005, α = 0.01 is pruned with and +/// without a table claiming relative error 0.004 at δ = 0.01; α = 0.005 +/// is kept in both. +#[test] +fn measured_accuracy_does_not_rescue_a_ddsketch_alpha() { + let alpha_01 = SketchParams::DDSketch { alpha: 0.01 }; + let alpha_005 = SketchParams::DDSketch { alpha: 0.005 }; + let measured = measured_p99(alpha_01.clone(), AccuracyMetric::Relative, 0.004); + let without = |memo| pass2(memo, &[entry(60_000)]); + let with = |memo| pass2_with_evidence(memo, &[entry(60_000)], &measured); + for (name, run) in [ + ("without the table", &without as &dyn Fn(Memo) -> _), + ("with the table", &with), + ] { + let (memo, state, readout) = + p99_with_per_value_target(SketchAlgorithm::DDSketch, AccuracyMetric::Relative, 0.005); + let logical = run(memo); + assert!( + pruned_below_required_accuracy(&logical, build_with(&logical, state, &alpha_01)), + "{name}: α = 0.01 is below the required accuracy" + ); + let into_readout = edges_into(&logical, own_candidate(readout), 0); + assert!( + into_readout.contains(&(build_with(&logical, state, &alpha_005), true)), + "{name}: {into_readout:?}" + ); + } +} diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index c698b99cf..b0518694b 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -213,7 +213,9 @@ each operator's error contract (`ErrorContract`, gathered per memo state in `ErrorMetricsOfMemo`) and the comparison of a bound with a target (`within_limit`); the deployment's measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a theoretical -guarantee. Only a sketch readout introduces error. Every other operator +guarantee, except for a θ whose theoretical guarantee can be deterministic +(DDSketch's relative error α): a measurement never replaces a guarantee that +holds with failure probability zero. Only a sketch readout introduces error. Every other operator passes its inputs' errors on through a contract, and the contracts are conservative: an operator with no contract for a metric fails the edge of an input whose error is in that metric (`RootPruneReason::MetricRule`), so an @@ -471,7 +473,11 @@ cheapest readouts (cost 2) misses B's target, so it bounds nothing. `AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the claimed guarantee applies: sources, filters, grouping, evaluation windows and all merged tumbling windows. Evidence for a narrower population cannot silently certify -a wider one. Data-workload facts are declared: the planner has no clock, so the caller +a wider one. `MeasuredSketchAccuracy`, the measured table, is keyed per θ +(sketch parameters, statistic, metric) and does not depend on the population: +each recorded error is the worst over the merge counts the calibration +measured. That it covers the populations the deployment plans for is the +deployment's responsibility, not checked by the planner. Data-workload facts are declared: the planner has no clock, so the caller drops facts that no longer hold before planning. Source contracts are assertions that the source or deployment must establish @@ -564,6 +570,8 @@ The `asap_accuracy` crate separates these responsibilities: accuracy/src/ ├── lib.rs # Re-exports: the public interfaces ├── evidence.rs # AccuracyEvidence: the deployment's measured accuracy +├── measured.rs # MeasuredSketchAccuracy: the measured table per θ, +│ # statistic and metric; its validation and lookup ├── error_contract/ # ErrorContract: each operator's error contract per │ # metric ├── metric_rule/ # ErrorMetricsOfMemo: the contracts, allowed metrics and diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 4565186bb..6930cd895 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -80,7 +80,7 @@ same pipeline behind the `OptimizationPass` trait. | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state/mod.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | | Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R), and with each query's exact-only reads $a_{q,e}$ and (X) (`milp/`, `exact_only.rs`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | -| The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | +| The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `MeasuredSketchAccuracy`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `measured.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | | The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | | What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities/`; Stage 2 reads it as the one list of exclusions it expands into (`DeploymentCapabilities::exclusions`); Pass 1 reads the same list only to skip a summary whose sketch algorithm one exclusion rules out outright (a `summary_family`-only exclusion); Stage 3 reads none of it | | The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `models.rs` (`PlanningModels`) and `pass.rs` (the trait); the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 7c5ea1b00..482cb3eca 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -62,7 +62,8 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- - A summary's guarantee is the deployment's measurement (`AccuracyEvidence::measured_guarantee`) when there is one, else its - algorithm's error formula (`theoretical_guarantee`); Pass 2 compares it + algorithm's error formula (`theoretical_guarantee`); a θ whose formula can + be deterministic (DDSketch's relative error α) keeps its formula; Pass 2 compares it with each readout's `RequiredAccuracy` (`ErrorMetricsOfMemo::edge_meets_logical_requirements`). It fails closed: no rule means no guarantee, and the edge is not admissible. diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 39f98ece6..2a76a0edb 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -194,10 +194,10 @@ asap_stage3_plan_selection::select_plan( | Input or output | Meaning | | --- | --- | -| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | +| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees, e.g. `asap_accuracy::MeasuredSketchAccuracy` loaded from a measured table and validated; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | | `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | | `DeploymentCapabilities` (`asap_types::deployment_capabilities`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries), `raw_data_retained` (default `true`; when `false`, stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`), `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains), `counter_semantics` (default: resets handled and extrapolated), `state_not_mergeable_across_windows` (default none) and `excluded_candidates` (`CandidateExclusion`s, default none). Every capability left out expands into exclusions (`DeploymentCapabilities::exclusions`, expanded once by the facade): Pass 1 skips the summaries they rule out outright, and Stage 2 applies them as one filter, pruning each matched candidate with `PruneReason::ExcludedByDeployment(source)`; with no plan left, Stage 3's rejection is `RejectionCause::ExcludedByDeployment` | -| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`, `raw_batch_width_ms`: the time one batch of raw rows covers in the data store, default 10,000, an assumed value), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | +| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`, `raw_batch_width_ms`: the time one batch of raw rows covers in the data store, default 10,000, an assumed value), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; an unknown field is refused). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory}` chooses among the candidates that fit |