From 5bac811a29bf3b190469ea03b5a9867aeb539220 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:51:25 +0000 Subject: [PATCH 1/3] test(accuracy, cost, devtools): measured sketch accuracy table and removed cost-table formats (tests only) Tests for MeasuredSketchAccuracy as Pass 2's AccuracyEvidence (exact parameters, the readout's own metric, largest recorded failure probability at most the request, deterministic guarantees never replaced), for refusing the cost table's provenance field and legacy array format, and for convert_rqe_atomic_costs writing the object form. Takes over the library tests of the asap_run draft. Stubs: measured.rs (todo!), and the converter ends in todo! after validating rows. --- Cargo.lock | 1 + crates/accuracy/Cargo.toml | 1 + crates/accuracy/src/lib.rs | 8 +- crates/accuracy/src/measured.rs | 97 +++++ .../tests/measured_sketch_accuracy.rs | 372 ++++++++++++++++++ crates/cost/src/node/usage/tests.rs | 145 +++++++ crates/cost/tests/measured_operation_costs.rs | 261 ++++++++++++ .../src/bin/convert_rqe_atomic_costs.rs | 7 +- crates/devtools/src/rqe_atomic_costs.rs | 127 +++--- .../src/test_fixtures.rs | 14 +- .../src/tests/edges.rs | 150 ++++++- 11 files changed, 1122 insertions(+), 61 deletions(-) create mode 100644 crates/accuracy/src/measured.rs create mode 100644 crates/accuracy/tests/measured_sketch_accuracy.rs create mode 100644 crates/cost/tests/measured_operation_costs.rs diff --git a/Cargo.lock b/Cargo.lock index 0117bfb7b..703605ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,7 @@ version = "0.1.0" dependencies = [ "asap-types", "asap_sketchlib", + "serde", "serde_json", ] diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 666477f7e..7d2da01b9 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } +serde = { version = "1", features = ["derive"] } serde_json = "1" [dev-dependencies] diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 3b049a6f9..075ac794c 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -2,7 +2,8 @@ //! theoretical guarantee of reading a statistic from a summary with given //! parameters ([`theoretical_guarantee`], from the per-algorithm //! guarantees in `guarantee`), the deployment's measured accuracy -//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract +//! ([`AccuracyEvidence`], in `evidence`, and the measured table +//! [`MeasuredSketchAccuracy`], in `measured`), each operator's error contract //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the //! metrics each query root allows, [`query_roots_with_target`], and the @@ -20,6 +21,7 @@ mod error_contract; mod evidence; mod guarantee; mod logical_edge; +mod measured; mod metric_rule; mod readout; mod readout_caps; @@ -28,5 +30,9 @@ pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::{tolerated_limit, within_limit}; +pub use measured::{ + ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy, + MeasuredSketchAccuracyError, +}; pub use metric_rule::{query_roots_with_target, ErrorMetricsOfMemo}; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/measured.rs b/crates/accuracy/src/measured.rs new file mode 100644 index 000000000..eac86804f --- /dev/null +++ b/crates/accuracy/src/measured.rs @@ -0,0 +1,97 @@ +//! The deployment's measured sketch accuracy: the errors a calibration run +//! measured on the executor's own sketches, which Pass 2 reads through +//! [`AccuracyEvidence`] before each summary's theoretical guarantee. +//! +//! Example: KLL with `k = 800`, read for `Quantile { q: 0.99 }`, measured +//! over 1,000 trials, has rank error 0.00176 at failure probability 0.01 +//! and 0.00121 at 0.05. A request at failure probability 0.02 gets +//! 0.00176 (the largest recorded failure probability at most 0.02); a +//! request at 0.001 gets no measured guarantee, so the theoretical one +//! applies. + +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchParams, SketchStatistic}; +use serde::{Deserialize, Serialize}; + +use crate::AccuracyEvidence; + +/// Errors measured on the executor's own sketches. Default: empty, every +/// sketch keeps its theoretical guarantee. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredSketchAccuracy { + pub entries: Vec, +} + +/// One sketch configuration, read for one statistic, in one metric. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredAccuracyEntry { + /// The exact parameters measured; only these match. + pub sketch: SketchParams, + /// The readout, e.g. `Quantile { q: 0.99 }`; only the same statistic + /// matches. + pub statistic: SketchStatistic, + /// The metric every recorded error is in, e.g. `Rank` for KLL. + pub accuracy_metric: AccuracyMetric, + /// Independent trials; every recorded failure probability is at least + /// 3 / trials. + pub trials: u32, + /// Ascending failure probability; the error does not increase as the + /// failure probability grows. + pub error_at: Vec, +} + +/// One measured point: the error not exceeded except with probability +/// `failure_probability`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErrorAtFailureProbability { + pub failure_probability: f64, + /// The (1 − δ) quantile of the per-value errors, the worst over the + /// measured merge counts, in the entry's `accuracy_metric`. + pub error: f64, +} + +/// Why a measured accuracy table is not valid. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum MeasuredSketchAccuracyError { + /// Entry `entry` has a failure probability outside (0, 1] or below + /// 3 / trials. + FailureProbabilityOutOfRange { + entry: usize, + failure_probability: f64, + }, + /// Entry `entry` has a negative or non-finite error. + InvalidError { entry: usize, error: f64 }, + /// Entry `entry`'s failure probabilities are not ascending, or its + /// error increases as the failure probability grows. + NotMonotone { entry: usize }, + /// Entries `first` and `second` measure the same sketch, statistic and + /// metric. + DuplicateEntry { first: usize, second: usize }, +} + +impl MeasuredSketchAccuracy { + /// Check every entry: failure probabilities in (0, 1] and at least + /// 3 / trials, errors finite and non-negative, ascending failure + /// probability with non-increasing error, and no two entries for one + /// (sketch, statistic, metric). + pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> { + todo!("measured accuracy table: implement after the tests are approved") + } +} + +impl AccuracyEvidence for MeasuredSketchAccuracy { + fn measured_guarantee( + &self, + _summary_type: &FieldDataType, + _parameters: &SketchParams, + _statistic: &SketchStatistic, + _failure_probability: f64, + ) -> Option { + todo!("measured accuracy table: implement after the tests are approved") + } +} diff --git a/crates/accuracy/tests/measured_sketch_accuracy.rs b/crates/accuracy/tests/measured_sketch_accuracy.rs new file mode 100644 index 000000000..63b2a9a32 --- /dev/null +++ b/crates/accuracy/tests/measured_sketch_accuracy.rs @@ -0,0 +1,372 @@ +//! The measured accuracy table as Pass 2's [`AccuracyEvidence`]: a request +//! at failure probability δ takes the largest recorded δ' ≤ δ of an entry +//! with exactly its parameters, statistic and metric, and nothing else. +//! Every expected value is read off the fixture by hand. + +use asap_accuracy::{ + theoretical_guarantee, AccuracyEvidence, ErrorAtFailureProbability, MeasuredAccuracyEntry, + MeasuredSketchAccuracy, MeasuredSketchAccuracyError, +}; +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; + +fn kll() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +fn p99() -> SketchStatistic { + SketchStatistic::Quantile { q: 0.99 } +} + +fn point(failure_probability: f64, error: f64) -> ErrorAtFailureProbability { + ErrorAtFailureProbability { + failure_probability, + error, + } +} + +/// KLL k = 547 read for the 0.99 quantile, rank error over 1,000 trials: +/// 0.0031 at δ = 0.003, 0.0024 at 0.01, 0.0016 at 0.05. +fn kll_547_entry() -> MeasuredAccuracyEntry { + MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k: 547 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Rank, + trials: 1_000, + error_at: vec![ + point(0.003, 0.0031), + point(0.01, 0.0024), + point(0.05, 0.0016), + ], + } +} + +fn table() -> MeasuredSketchAccuracy { + MeasuredSketchAccuracy { + entries: vec![kll_547_entry()], + } +} + +/// The error, failure probability and metric of a guarantee. +fn parts(guarantee: &AccuracyGuarantee) -> (f64, f64, AccuracyMetric) { + ( + guarantee.spec.error.evaluate().unwrap(), + guarantee.spec.failure_probability.evaluate().unwrap(), + guarantee.spec.accuracy_metric, + ) +} + +/// δ = 0.01 takes 0.01's error; 0.02 takes 0.01's too (the largest +/// recorded δ' ≤ 0.02, not the nearest 0.05 above); 0.05 and 1.0 (no +/// requirement) take 0.05's. +#[test] +fn lookup_takes_largest_recorded_delta_at_most_request() { + let table = table(); + let at = |delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + }; + assert_eq!(at(0.01), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.02), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.05), (0.0016, 0.05, AccuracyMetric::Rank)); + assert_eq!(at(1.0), (0.0016, 0.05, AccuracyMetric::Rank)); +} + +/// δ = 0.002 is below every recorded δ': no measured guarantee, never +/// 0.0031 extrapolated downward. +#[test] +fn lookup_no_extrapolation_below_smallest() { + let guarantee = + table().measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002); + assert_eq!(guarantee, None); +} + +/// Where the table has no measurement (δ = 0.002 above), the theoretical +/// guarantee applies: KLL k = 547 has rank error 2.296 / 547^0.9723 = +/// 0.004998 at δ ≥ 0.01, scaled below 0.01 by z(1 − δ/2) / z(0.995): +/// 0.004998 · 3.0902 / 2.5758 = 0.005997 at δ = 0.002. +#[test] +fn closed_form_applies_below_the_measured_range() { + let guarantee = + theoretical_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002).unwrap(); + let (error, _, metric) = parts(&guarantee); + assert_eq!(metric, AccuracyMetric::Rank); + assert!((error - 0.005997).abs() < 5e-6, "{error}"); +} + +/// Only the exact parameters, statistic and metric match: k = 548, the +/// median, or the relative metric get nothing. +#[test] +fn lookup_exact_params_statistic_metric() { + let table = table(); + assert_eq!( + table.measured_guarantee(&kll(), &SketchParams::Kll { k: 548 }, &p99(), 0.01), + None + ); + assert_eq!( + table.measured_guarantee( + &kll(), + &SketchParams::Kll { k: 547 }, + &SketchStatistic::Quantile { q: 0.5 }, + 0.01 + ), + None + ); + let relative = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + accuracy_metric: AccuracyMetric::Relative, + ..kll_547_entry() + }], + }; + // An entry in another metric than the readout's own (Rank for KLL) is + // never returned. + assert_eq!( + relative.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.01), + None + ); +} + +/// An empty table measures nothing. +#[test] +fn empty_table_measures_nothing() { + let empty = MeasuredSketchAccuracy::default(); + for delta in [0.001, 0.01, 1.0] { + assert_eq!( + empty.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta), + None + ); + } +} + +/// Lookup monotonicity (I10): a larger δ never gets a larger measured +/// error than a smaller one. +#[test] +fn lookup_error_does_not_grow_with_delta() { + let table = table(); + let deltas = [0.003, 0.005, 0.01, 0.02, 0.05, 0.5, 1.0]; + let errors: Vec = deltas + .iter() + .map(|&delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + .0 + }) + .collect(); + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{errors:?}" + ); +} + +/// A deterministic sketch keeps its deterministic guarantee: a measured +/// (probabilistic) entry for DDSketch α = 0.01 is not returned. The rule +/// is general: the table is never read for a θ whose theoretical guarantee +/// has failure probability zero; DDSketch's relative α is the one such +/// case today. +#[test] +fn measured_entry_never_replaces_a_deterministic_guarantee() { + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: SketchParams::DDSketch { alpha: 0.01 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.004)], + }], + }; + let guarantee = table.measured_guarantee( + &FieldDataType::Sketch(SketchAlgorithm::DDSketch), + &SketchParams::DDSketch { alpha: 0.01 }, + &p99(), + 0.01, + ); + assert_eq!(guarantee, None); +} + +/// Whether a validation error is the expected one. +type IsReason = fn(&MeasuredSketchAccuracyError) -> bool; + +/// Each invalid table is rejected, for its own reason: the error variant +/// is asserted, so a table refused for the wrong reason fails the test. +#[test] +fn table_validation() { + use MeasuredSketchAccuracyError as E; + assert_eq!(table().validate(), Ok(())); + let with = |error_at: Vec| MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + error_at, + ..kll_547_entry() + }], + }; + let rejected: [(&str, MeasuredSketchAccuracy, IsReason); 8] = [ + // δ = 0.002 < 3 / 1,000 trials. + ( + "delta below 3/trials", + with(vec![point(0.002, 0.004)]), + |e| { + *e == E::FailureProbabilityOutOfRange { + entry: 0, + failure_probability: 0.002, + } + }, + ), + // The error grows with δ. + ( + "error increasing in delta", + with(vec![point(0.01, 0.002), point(0.05, 0.003)]), + |e| *e == E::NotMonotone { entry: 0 }, + ), + // δ not ascending. + ( + "unsorted deltas", + with(vec![point(0.05, 0.0016), point(0.01, 0.0024)]), + |e| *e == E::NotMonotone { entry: 0 }, + ), + ("delta zero", with(vec![point(0.0, 0.01)]), |e| { + *e == E::FailureProbabilityOutOfRange { + entry: 0, + failure_probability: 0.0, + } + }), + ("delta above one", with(vec![point(1.5, 0.01)]), |e| { + *e == E::FailureProbabilityOutOfRange { + entry: 0, + failure_probability: 1.5, + } + }), + ("negative error", with(vec![point(0.01, -1.0)]), |e| { + *e == E::InvalidError { + entry: 0, + error: -1.0, + } + }), + // NaN != NaN, so match the variant and check the error is NaN. + ( + "NaN error", + with(vec![point(0.01, f64::NAN)]), + |e| matches!(e, E::InvalidError { entry: 0, error } if error.is_nan()), + ), + ( + "duplicate entry", + MeasuredSketchAccuracy { + entries: vec![kll_547_entry(), kll_547_entry()], + }, + |e| { + *e == E::DuplicateEntry { + first: 0, + second: 1, + } + }, + ), + ]; + for (name, table, expected) in rejected { + let error = table.validate().expect_err(name); + assert!(expected(&error), "{name}: wrong reason {error:?}"); + } +} + +/// The table's JSON is the issue's example format, and round-trips. +#[test] +fn accuracy_json_round_trips() { + let text = r#"{"entries": [{"sketch": {"Kll": {"k": 800}}, "statistic": {"Quantile": {"q": 0.99}}, + "accuracy_metric": "rank", "trials": 1000, "error_at": [ + {"failure_probability": 0.01, "error": 0.00176}, + {"failure_probability": 0.05, "error": 0.00121}]}]}"#; + let table: MeasuredSketchAccuracy = serde_json::from_str(text).unwrap(); + assert_eq!(table.entries[0].sketch, SketchParams::Kll { k: 800 }); + assert_eq!(table.entries[0].error_at[1], point(0.05, 0.00121)); + let written = serde_json::to_string(&table).unwrap(); + assert_eq!( + serde_json::from_str::(&written).unwrap(), + table + ); +} + +/// A request at failure probability 0, below 0, or NaN has no recorded +/// δ′ ≤ δ: nothing is returned, and nothing panics. +#[test] +fn a_request_at_zero_or_nan_failure_probability_gets_nothing() { + let table = table(); + for delta in [0.0, -0.1, f64::NAN] { + assert_eq!( + table.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta), + None, + "δ = {delta}" + ); + } +} + +/// The returned guarantee states the recorded δ′ (0.01), not the requested +/// δ (0.02), so Pass 2's check that a guarantee holds at the requested +/// failure probability (0.01 ≤ 0.02) holds by construction. +#[test] +fn the_returned_guarantee_states_the_recorded_failure_probability() { + let guarantee = table() + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.02) + .unwrap(); + assert_eq!(guarantee.spec.failure_probability.evaluate(), Some(0.01)); + assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Rank); +} + +/// HLL's theoretical guarantee (relative standard error 1.04 / √m) has an +/// unknown failure probability, so Pass 2 prunes an HLL readout under a +/// target. A measured entry supplies one: HLL precision 13 read for the +/// distinct count, relative error 0.02 at δ = 0.01 over 1,000 trials, gives +/// the guarantee (0.02, 0.01, Relative). Measured evidence comes first, and +/// the deterministic rule does not apply (the theoretical failure +/// probability is unknown, not zero). The recorded error is the worst over +/// the merge counts the calibration measured, so the guarantee also holds +/// for a state merged from that many windows. +#[test] +fn a_measured_entry_supplies_a_guarantee_where_theory_states_no_failure_probability() { + let hll = FieldDataType::Sketch(SketchAlgorithm::Hll); + let parameters = SketchParams::Hll { precision: 13 }; + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: parameters.clone(), + statistic: SketchStatistic::Cardinality, + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.02)], + }], + }; + assert_eq!(table.validate(), Ok(())); + let guarantee = table + .measured_guarantee(&hll, ¶meters, &SketchStatistic::Cardinality, 0.01) + .unwrap(); + assert_eq!(parts(&guarantee), (0.02, 0.01, AccuracyMetric::Relative)); +} + +/// A readout with no metric of its own (UnivMon's distinct count has no +/// listed guarantee) never gets a measured one: there is no readout metric +/// for the entry's to match. +#[test] +fn an_entry_for_a_readout_with_no_metric_is_never_returned() { + let univmon = FieldDataType::Sketch(SketchAlgorithm::UnivMon); + let parameters = SketchParams::UnivMon { + heap_size: 32, + sketch_rows: 5, + sketch_cols: 1024, + layers: 8, + }; + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: parameters.clone(), + statistic: SketchStatistic::Cardinality, + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.05)], + }], + }; + assert_eq!( + table.measured_guarantee(&univmon, ¶meters, &SketchStatistic::Cardinality, 0.01), + None + ); +} diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 407ddb326..611aaf6be 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -3364,3 +3364,148 @@ fn group_visit_units_follow_the_raw_batch_width() { assert_eq!(visits(1_000), 300_000.0); assert_eq!(visits(10_000), 30_000.0); } + +/// A query-time producer of `rows` plain rows, for a node under test. +fn rows_of<'a>(producer: &'a PhysicalASAPDAGNode, rows: u64) -> ProducerOutput<'a> { + ProducerOutput { + payload: &producer.payload, + output_schema: &producer.output_schema, + output: EdgeStatistics { + rows, + bytes: rows * 16, + }, + sources: BTreeSet::new(), + family: StateFamily::None, + unchanged_columns: BTreeSet::new(), + tumbling_windows: 1, + chain_tumbling_window_width_ms: None, + state_input_rows: None, + raw_rows_span_ms: None, + } +} + +/// A one-minute `TimeRange` over node 0. +fn one_minute_range() -> PhysicalASAPDAGNode { + node(Payload::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: 0, + })) +} + +/// A query-time `SummaryAgg` of an exact sum over 600 input rows, with a +/// measured entry of 10 ns per row and 500 ns per computation, takes +/// 600 · 10 + 500 = 6,500 ns = 0.0065 ms per execution; computed once a +/// second, 6.5e-6 core-seconds per second, from the table. +#[test] +fn node_priced_by_hand() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::SummaryAgg, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 10.0, + ns_per_invocation: 500.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let build = node(Payload::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })); + let usage = node_resource_usage_and_cost( + &build, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.0065).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); + assert!( + (usage.cost.cpu_core_seconds_per_second - 6.5e-6).abs() < 1e-15, + "{}", + usage.cost.cpu_core_seconds_per_second + ); +} + +/// An `Aggregate` is priced from the table only when each of its measures +/// has an entry: with an entry for `sum` (10 ns per row) and none for +/// `quantile`, `Aggregate[sum, quantile]` keeps the formulas, and +/// `Aggregate[sum]` over 600 rows takes 600 · 10 = 6,000 ns. +#[test] +fn aggregate_needs_every_measure() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("sum".into()), + ns_per_unit: 10.0, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let aggregate = |measures: Vec| { + node(Payload::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::none()), + output_names: vec![String::new(); measures.len()], + filters: vec![None; measures.len()], + measures, + having: None, + child: 0, + })) + }; + let both = aggregate(vec![ + AggIntent::Sum { col: None }, + AggIntent::Quantile { col: None, q: 0.5 }, + ]); + let usage = node_resource_usage_and_cost( + &both, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Analytical); + let sum = aggregate(vec![AggIntent::Sum { col: None }]); + let usage = node_resource_usage_and_cost( + &sum, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.006).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); +} diff --git a/crates/cost/tests/measured_operation_costs.rs b/crates/cost/tests/measured_operation_costs.rs new file mode 100644 index 000000000..4293ab13a --- /dev/null +++ b/crates/cost/tests/measured_operation_costs.rs @@ -0,0 +1,261 @@ +//! The measured cost table: [`lookup`] (exact match, interpolation in log +//! size, phase shadowing, no extrapolation), its JSON round trip and the +//! formats the `asap_run` design deletes. Every expected price is +//! computed by hand from the entries below. + +use asap_cost::{ + lookup, CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind, StateFamily, +}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::summary_type::{ExactKind, SketchParams}; + +/// A sketch's size for interpolation: KLL's `k`, 1 for any other sketch. +fn size_of(params: &SketchParams) -> f64 { + match params { + SketchParams::Kll { k } => *k as f64, + _ => 1.0, + } +} + +/// A `SummaryAgg` entry for KLL with `k` at `phase`. +fn kll_entry( + k: u32, + phase: Option, + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +) -> OperatorCost { + OperatorCost { + operator: OperatorKind::SummaryAgg, + phase, + sketch: Some(SketchParams::Kll { k }), + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation, + memory_bytes_per_state, + } +} + +/// KLL `k = 200` at 100 ns/unit, 1,000 ns/invocation, 2,000 bytes, and +/// `k = 800` at 300 ns/unit, 3,000 ns/invocation, 8,000 bytes, for both +/// phases. +fn kll_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, Some(8_000.0)), + ], + ..MeasuredOperationCosts::default() + } +} + +/// A price's fields: ns per unit, ns per invocation, bytes per state. +#[derive(Debug, PartialEq)] +struct Price { + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +} + +fn kll_price_at(table: &MeasuredOperationCosts, phase: ExecutionTiming, k: u32) -> Option { + lookup( + table, + OperatorKind::SummaryAgg, + phase, + StateFamily::Sketch(&SketchParams::Kll { k }), + size_of, + ) + .map(|price| Price { + ns_per_unit: price.ns_per_unit, + ns_per_invocation: price.ns_per_invocation, + memory_bytes_per_state: price.memory_bytes_per_state, + }) +} + +fn assert_close(actual: f64, expected: f64) { + assert!( + (actual - expected).abs() <= 1e-9 * expected.abs().max(1.0), + "{actual} != {expected}" + ); +} + +/// An entry with exactly the node's parameters is its price, not an +/// interpolation. +#[test] +fn lookup_exact_params_win() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 200).unwrap(); + assert_eq!(price.ns_per_unit, 100.0); + assert_eq!(price.ns_per_invocation, 1_000.0); + assert_eq!(price.memory_bytes_per_state, Some(2_000.0)); +} + +/// k = 400 lies at ln(400/200) / ln(800/200) = 0.5 between the two +/// entries in log size: 100 + 0.5·200 = 200 ns/unit, 2,000 ns/invocation, +/// 5,000 bytes. Linear in size would give 166.7 ns/unit. +#[test] +fn lookup_interpolates_in_log_size() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 400).unwrap(); + assert_close(price.ns_per_unit, 200.0); + assert_close(price.ns_per_invocation, 2_000.0); + assert_close(price.memory_bytes_per_state.unwrap(), 5_000.0); +} + +/// Below the smallest and above the largest measured size, the nearest +/// entry's price: 100 at k = 100 (not 33), 300 at k = 1600 (not 367). +#[test] +fn lookup_no_extrapolation() { + let table = kll_table(); + let below = kll_price_at(&table, ExecutionTiming::QueryTime, 100).unwrap(); + let above = kll_price_at(&table, ExecutionTiming::QueryTime, 1_600).unwrap(); + assert_eq!(below.ns_per_unit, 100.0); + assert_eq!(above.ns_per_unit, 300.0); +} + +/// A query-time entry for k = 200 at 999 ns/unit hides both both-phase +/// entries at query time: k = 400 has only k = 200 to go by, so 999. At +/// ingestion time there is no ingestion entry, so the both-phase pair +/// interpolates to 200. +#[test] +fn lookup_phase_entries_shadow_both() { + let mut table = kll_table(); + table.operators.push(kll_entry( + 200, + Some(ExecutionTiming::QueryTime), + 999.0, + 0.0, + None, + )); + let query = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + let ingestion = kll_price_at(&table, ExecutionTiming::IngestionTime, 400).unwrap(); + assert_eq!(query.ns_per_unit, 999.0); + assert_close(ingestion.ns_per_unit, 200.0); +} + +/// With no bytes on k = 800, k = 400 has one memory neighbour only: no +/// measured bytes (the formulas' bytes apply), never an interpolation +/// with 0. +#[test] +fn lookup_memory_needs_both_neighbours() { + let table = MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, None), + ], + ..MeasuredOperationCosts::default() + }; + let price = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + assert_eq!(price.memory_bytes_per_state, None); +} + +/// KLL entries do not price a DDSketch node. +#[test] +fn lookup_other_algorithm_misses() { + let price = lookup( + &kll_table(), + OperatorKind::SummaryAgg, + ExecutionTiming::QueryTime, + StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), + size_of, + ); + assert!(price.is_none()); +} + +/// A table with one of everything a JSON file can hold. +fn full_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry( + 200, + Some(ExecutionTiming::IngestionTime), + 100.5, + 1_000.25, + Some(2_048.0), + ), + OperatorCost { + operator: OperatorKind::SummaryMerge, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 3.75, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("quantile".into()), + ns_per_unit: 12.0, + ns_per_invocation: 250.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: 20.0, + ns_per_invocation: 5_000.0, + memory_bytes_per_state: None, + }, + ], + ns_per_evaluation: 50_000.0, + ..MeasuredOperationCosts::default() + } +} + +/// `costs.json` round-trips: written, read back and written again it is +/// byte-equal, and it validates. +#[test] +fn costs_json_round_trips() { + let table = full_table(); + let written = serde_json::to_string_pretty(&table).unwrap(); + let read: MeasuredOperationCosts = serde_json::from_str(&written).unwrap(); + assert_eq!(read, table); + assert_eq!(serde_json::to_string_pretty(&read).unwrap(), written); + let inputs = CostInputs { + measured_operation_costs: read, + ..CostInputs::default() + }; + inputs.validate().unwrap(); +} + +/// `provenance` is deleted (where a table was measured is recorded by the +/// calibration run, not in the table) and so is the array format of +/// earlier versions: a table carrying either no longer loads. +#[test] +fn costs_json_rejects_removed_fields() { + let with_provenance = r#"{"operators": [], "ns_per_evaluation": 0.0, + "provenance": {"executor_version": "0.1.0", "machine": "x86_64"}}"#; + assert!( + serde_json::from_str::(with_provenance).is_err(), + "a table with `provenance` must not load" + ); + let legacy_array = r#"[{"sketch": {"Kll": {"k": 200}}, "operation": "update", + "cpu_ns_per_operation": 100.0}]"#; + assert!( + serde_json::from_str::(legacy_array).is_err(), + "a table in the array format must not load" + ); +} + +/// A `CostInputs` file (the `asap_planner --cost-inputs` format) whose +/// measured table still carries `provenance` is refused, and the message +/// names the field, so a stale file fails loudly instead of loading with +/// the field ignored. +#[test] +fn a_cost_inputs_file_with_provenance_is_refused() { + let mut inputs = serde_json::to_value(CostInputs::default()).unwrap(); + inputs["measured_operation_costs"] = serde_json::json!({ + "operators": [], + "ns_per_evaluation": 0.0, + "provenance": {"executor_version": "0.1.0", "machine": "x86_64"} + }); + let error = serde_json::from_value::(inputs) + .expect_err("a cost-inputs file with `provenance` must not load"); + assert!(error.to_string().contains("provenance"), "{error}"); +} diff --git a/crates/devtools/src/bin/convert_rqe_atomic_costs.rs b/crates/devtools/src/bin/convert_rqe_atomic_costs.rs index a08cdcca8..949b4d485 100644 --- a/crates/devtools/src/bin/convert_rqe_atomic_costs.rs +++ b/crates/devtools/src/bin/convert_rqe_atomic_costs.rs @@ -2,9 +2,8 @@ // optimizer_cost/rqe_atomic_costs.json measured_operation_costs.json // // Reads sketch-bench's per-operation cost table (`rqe_atomic_costs.json`) -// and writes plan selection's measured table: a JSON array of -// `{"sketch": , "operation": "update|merge|estimate", -// "cpu_ns_per_operation": }`, three entries per converted row +// and writes plan selection's measured table in its object form +// (`MeasuredOperationCosts`): four `OperatorCost` entries per converted row // (`asap_devtools::rqe_atomic_costs`). Set it as // `CostInputs::measured_operation_costs` (`asap_planner --cost-inputs`). Skipped rows (exact // accumulators, Hydra structures, unknown names) are listed on stderr. @@ -27,7 +26,7 @@ fn main() -> Result<(), Box> { )?; eprintln!( "wrote {} entries to {output}", - converted.measured_operation_costs.len() + converted.measured_operation_costs.operators.len() ); Ok(()) } diff --git a/crates/devtools/src/rqe_atomic_costs.rs b/crates/devtools/src/rqe_atomic_costs.rs index faccfed55..78f776577 100644 --- a/crates/devtools/src/rqe_atomic_costs.rs +++ b/crates/devtools/src/rqe_atomic_costs.rs @@ -9,12 +9,16 @@ //! "insert_cpu_secs": 8.5e-8, "merge_cpu_secs": 1.2e-6, "query_cpu_secs": 4.0e-7, ...} //! ``` //! -//! and each becomes three entries, one per operation, in CPU nanoseconds: +//! and each becomes four `OperatorCost` entries of the table's object form, +//! in CPU nanoseconds per unit, for both phases: //! //! ```json -//! [{"sketch": {"Kll": {"k": 200}}, "operation": "update", "cpu_ns_per_operation": 85.0}, -//! {"sketch": {"Kll": {"k": 200}}, "operation": "merge", "cpu_ns_per_operation": 1200.0}, -//! {"sketch": {"Kll": {"k": 200}}, "operation": "estimate", "cpu_ns_per_operation": 400.0}] +//! {"operators": [ +//! {"operator": "SummaryAgg", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 85.0, "ns_per_invocation": 0.0}, +//! {"operator": "SummaryTumblingWindowChain", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 85.0, "ns_per_invocation": 0.0}, +//! {"operator": "SummaryMerge", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 1200.0, "ns_per_invocation": 0.0}, +//! {"operator": "SummaryEstimate", "phase": null, "sketch": {"Kll": {"k": 200}}, "ns_per_unit": 400.0, "ns_per_invocation": 0.0}], +//! "ns_per_evaluation": 0.0} //! ``` //! //! Only this converter reads the sketch-bench format; the planner reads the @@ -24,7 +28,7 @@ //! //! [`CostInputs::measured_operation_costs`]: asap_cost::CostInputs::measured_operation_costs -use asap_cost::{MeasuredOperationCost, SummaryOperation}; +use asap_cost::{MeasuredOperationCost, MeasuredOperationCosts, SummaryOperation}; use asap_types::summary_type::SketchParams; use serde_json::Value; @@ -40,9 +44,11 @@ pub struct SkippedRow { /// What [`convert_rqe_atomic_costs`] produced. #[derive(Debug, Clone, PartialEq)] pub struct ConvertedCosts { - /// The measured table: three entries per converted row (update, merge, - /// estimate), in row order. - pub measured_operation_costs: Vec, + /// The measured table in its object form: per converted row, an update + /// becomes `SummaryAgg` and `SummaryTumblingWindowChain` entries, a merge a + /// `SummaryMerge` entry and an estimate a `SummaryEstimate` entry, each + /// for both phases, in row order. + pub measured_operation_costs: MeasuredOperationCosts, /// The rows skipped, in row order. pub skipped_rows: Vec, } @@ -55,10 +61,11 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result let rows = rows .as_array() .ok_or("rqe_atomic_costs.json is not a JSON array of rows")?; - let mut converted = ConvertedCosts { - measured_operation_costs: Vec::new(), - skipped_rows: Vec::new(), - }; + // Test-first stub: the rows are still converted into the earlier + // format's entries, which the implementation replaces by `OperatorCost` + // entries. + let mut legacy_entries: Vec = Vec::new(); + let mut skipped_rows = Vec::new(); for (index, row) in rows.iter().enumerate() { let sketch_name = row["sketch"] .as_str() @@ -67,7 +74,7 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result let sketch = match sketch_params(sketch_name, params) { Ok(Some(sketch)) => sketch, Ok(None) => { - converted.skipped_rows.push(SkippedRow { + skipped_rows.push(SkippedRow { sketch_name: sketch_name.to_string(), reason: skip_reason(sketch_name).to_string(), }); @@ -75,11 +82,7 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result } Err(error) => return Err(format!("row {index} ({sketch_name}): {error}")), }; - if converted - .measured_operation_costs - .iter() - .any(|entry| entry.sketch == sketch) - { + if legacy_entries.iter().any(|entry| entry.sketch == sketch) { return Err(format!( "row {index} ({sketch_name}): a second row for {sketch:?}" )); @@ -95,16 +98,15 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result .ok_or(format!( "row {index} ({sketch_name}): `{field}` is not a non-negative number" ))?; - converted - .measured_operation_costs - .push(MeasuredOperationCost { - sketch: sketch.clone(), - operation, - cpu_ns_per_operation: seconds * 1e9, - }); + legacy_entries.push(MeasuredOperationCost { + sketch: sketch.clone(), + operation, + cpu_ns_per_operation: seconds * 1e9, + }); } } - Ok(converted) + let _ = (legacy_entries, skipped_rows); + todo!("write the object form: OperatorCost entries, phase None") } /// Why a row whose name has no planner sketch is skipped. @@ -183,10 +185,14 @@ mod tests { /// Four sketch-bench rows in the format of `rqe_atomic_costs.json`: a /// KLL and a Count-Min top-k convert, the exact sum and the Hydra HLL - /// are skipped. The table written as JSON reads back as the same - /// entries, with seconds turned into nanoseconds. + /// are skipped. Each converted row gives four `OperatorCost` entries for + /// both phases (an update prices `SummaryAgg` and + /// `SummaryTumblingWindowChain`, a merge `SummaryMerge`, an estimate + /// `SummaryEstimate`), with seconds turned into nanoseconds per unit; + /// the table written as JSON reads back equal. #[test] fn a_small_sample_converts_and_round_trips() { + use asap_cost::{OperatorCost, OperatorKind}; let sample = json!([ {"sketch": "kll-percall", "sketch_config": {"algorithm": "kll-percall", "params": {"k": 200}}, "mem_bytes_per_instance": 9648.0, "insert_cpu_secs": 8.5e-8, "merge_cpu_secs": 1.2e-6, @@ -208,29 +214,44 @@ mod tests { depth: 3, heap_size: 32, }; - let expected: Vec<_> = [ - (&kll, SummaryOperation::Update, 85.0), - (&kll, SummaryOperation::Merge, 1200.0), - (&kll, SummaryOperation::Estimate, 400.0), - (&cms, SummaryOperation::Update, 60.0), - (&cms, SummaryOperation::Merge, 5000.0), - (&cms, SummaryOperation::Estimate, 350.0), - ] - .into_iter() - .map(|(sketch, operation, ns)| (sketch.clone(), operation, ns)) - .collect(); - let got: Vec<_> = converted - .measured_operation_costs - .iter() - .map(|e| (e.sketch.clone(), e.operation, e.cpu_ns_per_operation)) - .collect(); - assert_eq!(got.len(), expected.len()); - for ((sketch, operation, ns), (want_sketch, want_operation, want_ns)) in - got.iter().zip(&expected) - { - assert_eq!((sketch, operation), (want_sketch, want_operation)); - assert!((ns - want_ns).abs() < 1e-9 * want_ns, "{ns} vs {want_ns}"); + let expected = [ + (&kll, OperatorKind::SummaryAgg, 85.0), + (&kll, OperatorKind::SummaryTumblingWindowChain, 85.0), + (&kll, OperatorKind::SummaryMerge, 1200.0), + (&kll, OperatorKind::SummaryEstimate, 400.0), + (&cms, OperatorKind::SummaryAgg, 60.0), + (&cms, OperatorKind::SummaryTumblingWindowChain, 60.0), + (&cms, OperatorKind::SummaryMerge, 5000.0), + (&cms, OperatorKind::SummaryEstimate, 350.0), + ]; + let table = &converted.measured_operation_costs; + assert_eq!(table.operators.len(), expected.len()); + for (entry, (sketch, operator, ns)) in table.operators.iter().zip(expected) { + let OperatorCost { + operator: got_operator, + phase, + sketch: got_sketch, + exact, + measure, + ns_per_unit, + ns_per_invocation, + memory_bytes_per_state, + } = entry; + assert_eq!( + (got_operator, got_sketch.as_ref(), *phase), + (&operator, Some(sketch), None) + ); + assert_eq!( + (exact, measure, memory_bytes_per_state), + (&None, &None, &None) + ); + assert_eq!(*ns_per_invocation, 0.0); + assert!( + (ns_per_unit - ns).abs() < 1e-9 * ns, + "{ns_per_unit} vs {ns}" + ); } + assert_eq!(table.ns_per_evaluation, 0.0); let skipped: Vec<_> = converted .skipped_rows .iter() @@ -238,9 +259,9 @@ mod tests { .collect(); assert_eq!(skipped, ["exact-sum", "hydra-hll"]); - let written = serde_json::to_string(&converted.measured_operation_costs).unwrap(); - let read: Vec = serde_json::from_str(&written).unwrap(); - assert_eq!(read, converted.measured_operation_costs); + let written = serde_json::to_string(table).unwrap(); + let read: MeasuredOperationCosts = serde_json::from_str(&written).unwrap(); + assert_eq!(&read, table); } /// Bad input is an error, not a skipped row: a missing time, a negative diff --git a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs index f0e48388d..90e2293d9 100644 --- a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs @@ -5,7 +5,7 @@ //! its output. use std::collections::BTreeSet; -use asap_accuracy::NoMeasuredGuarantees; +use asap_accuracy::{AccuracyEvidence, NoMeasuredGuarantees}; use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; @@ -59,7 +59,17 @@ pub fn entry(interval_ms: u32) -> QueryWorkloadEntry { } pub fn pass2(memo: Memo, entries: &[QueryWorkloadEntry]) -> LogicalASAPMemoCandidates { - generate_logical_asap_memo_candidates(memo, entries, &NoMeasuredGuarantees).unwrap() + pass2_with_evidence(memo, entries, &NoMeasuredGuarantees) +} + +/// [`pass2`] with the deployment's measured accuracy `evidence`, read +/// before each summary's theoretical guarantee. +pub fn pass2_with_evidence( + memo: Memo, + entries: &[QueryWorkloadEntry], + evidence: &dyn AccuracyEvidence, +) -> LogicalASAPMemoCandidates { + generate_logical_asap_memo_candidates(memo, entries, evidence).unwrap() } /// The edges into input `input` of `consumer`: each producer and α^log. diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs index fe726caf0..b4ed7370d 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs @@ -1,7 +1,8 @@ //! Steps 4 to 6: the logical edges Pass 2 assembles and keeps ($\alpha^{\log}$ //! and the metric rule themselves are tested in `asap_accuracy`). -use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_accuracy::{ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy}; +use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ @@ -120,3 +121,150 @@ fn a_merge_is_judged_by_the_parameters_it_carries() { } if max_error == 0.005 ))); } + +/// A measured accuracy table with one entry: `sketch` read for the 0.99 +/// quantile in `accuracy_metric`, over 1,000 trials, with `error` at +/// failure probability 0.01. +fn measured_p99( + sketch: SketchParams, + accuracy_metric: AccuracyMetric, + error: f64, +) -> MeasuredSketchAccuracy { + MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric, + trials: 1_000, + error_at: vec![ErrorAtFailureProbability { + failure_probability: 0.01, + error, + }], + }], + } +} + +/// `quantile_over_time(0.99, m[5m])` over one series of `m` (so δ is not +/// split across readout values), with a per-value `metric` target of +/// `max_error` at δ = 0.01: the memo, its state group and readout group. +fn p99_with_per_value_target( + algorithm: SketchAlgorithm, + metric: AccuracyMetric, + max_error: f64, +) -> (Memo, MemoGroupId, MemoGroupId) { + let mut memo = Memo::new(&with_series("m", 1)); + let (_, state, readout) = quantile_over_time(&mut memo, algorithm, 0.99, 300); + query( + &mut memo, + readout, + Some( + AccuracyTarget::max_error(max_error) + .failure_probability(0.01) + .accuracy_metric(metric) + .accuracy_scope(AccuracyScope::PerValue), + ), + ); + (memo, state, readout) +} + +/// The build candidate of `state` with parameters `parameters`. +fn build_with( + logical: &crate::LogicalASAPMemoCandidates, + state: MemoGroupId, + parameters: &SketchParams, +) -> CandidateForMemoGroupId { + let index_in_group = candidates_of(&logical.memo, state) + .iter() + .position(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryAgg { .. }) + ) && c.logical_properties.parameters() == Some(parameters) + }) + .unwrap_or_else(|| panic!("a build with {parameters:?} in the state group")); + CandidateForMemoGroupId { + group: state, + index_in_group, + } +} + +/// Whether `candidate` is pruned, every cause `BelowRequiredAccuracy`. +fn pruned_below_required_accuracy( + logical: &crate::LogicalASAPMemoCandidates, + candidate: CandidateForMemoGroupId, +) -> bool { + pruned_entry(logical, candidate).is_some_and(|pruned| { + !pruned.causes.is_empty() + && pruned + .causes + .iter() + .all(|cause| matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. })) + }) +} + +/// Measured accuracy keeps a smaller KLL that theory prunes. Rank target +/// 0.0025 per value at δ = 0.01: in theory KLL k = 547 has rank error +/// 2.296 / 547^0.9723 = 0.004998 > 0.0025 and is pruned, k = 1,404 has +/// 0.00200 and is kept. A table measuring k = 547 at rank error 0.0024 +/// at δ′ = 0.01 (≤ 0.0025) keeps the k = 547 edge. +#[test] +fn measured_accuracy_keeps_a_smaller_kll_that_theory_prunes() { + let k547 = SketchParams::Kll { k: 547 }; + let k1404 = SketchParams::Kll { k: 1404 }; + + let (memo, state, readout) = + p99_with_per_value_target(SketchAlgorithm::Kll, AccuracyMetric::Rank, 0.0025); + let theory = pass2(memo, &[entry(60_000)]); + assert!( + pruned_below_required_accuracy(&theory, build_with(&theory, state, &k547)), + "without evidence, k = 547 is below the required accuracy" + ); + let into_readout = edges_into(&theory, own_candidate(readout), 0); + assert!( + into_readout.contains(&(build_with(&theory, state, &k1404), true)), + "{into_readout:?}" + ); + + let (memo, state, readout) = + p99_with_per_value_target(SketchAlgorithm::Kll, AccuracyMetric::Rank, 0.0025); + let measured = measured_p99(k547.clone(), AccuracyMetric::Rank, 0.0024); + let with_table = pass2_with_evidence(memo, &[entry(60_000)], &measured); + let build_547 = build_with(&with_table, state, &k547); + assert!(pruned_entry(&with_table, build_547).is_none()); + let into_readout = edges_into(&with_table, own_candidate(readout), 0); + assert!( + into_readout.contains(&(build_547, true)), + "{into_readout:?}" + ); +} + +/// Measured accuracy never rescues a DDSketch α: its relative guarantee is +/// deterministic (failure probability zero), so the table is not read. At +/// a per-value relative target of 0.005, α = 0.01 is pruned with and +/// without a table claiming relative error 0.004 at δ = 0.01; α = 0.005 +/// is kept in both. +#[test] +fn measured_accuracy_does_not_rescue_a_ddsketch_alpha() { + let alpha_01 = SketchParams::DDSketch { alpha: 0.01 }; + let alpha_005 = SketchParams::DDSketch { alpha: 0.005 }; + let measured = measured_p99(alpha_01.clone(), AccuracyMetric::Relative, 0.004); + let without = |memo| pass2(memo, &[entry(60_000)]); + let with = |memo| pass2_with_evidence(memo, &[entry(60_000)], &measured); + for (name, run) in [ + ("without the table", &without as &dyn Fn(Memo) -> _), + ("with the table", &with), + ] { + let (memo, state, readout) = + p99_with_per_value_target(SketchAlgorithm::DDSketch, AccuracyMetric::Relative, 0.005); + let logical = run(memo); + assert!( + pruned_below_required_accuracy(&logical, build_with(&logical, state, &alpha_01)), + "{name}: α = 0.01 is below the required accuracy" + ); + let into_readout = edges_into(&logical, own_candidate(readout), 0); + assert!( + into_readout.contains(&(build_with(&logical, state, &alpha_005), true)), + "{name}: {into_readout:?}" + ); + } +} From 51cbe12449ca9f61941100710738f7d8ce65ffc8 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:07:24 +0000 Subject: [PATCH 2/3] feat(accuracy, cost, devtools): MeasuredSketchAccuracy as Pass 2's evidence; delete the cost table's provenance and legacy array format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MeasuredSketchAccuracy returns, for exactly the requested sketch parameters and statistic in the readout's own metric, the error at the largest recorded failure probability at most the request, stating that recorded failure probability; never extrapolated, and never for a θ whose theoretical guarantee can be deterministic (DDSketch). Each recorded error is the worst case over the merge counts measured. MeasuredOperationCosts derives Deserialize with deny_unknown_fields: provenance, MeasuredOperationCost, SummaryOperation and the array format are deleted. convert_rqe_atomic_costs writes OperatorCost entries itself. Test updates for main: the closed-form test passes the sign and distinct bound arguments theoretical_guarantee now takes; the Pass 2 fixture declares non-negative values, which DDSketch's guarantee now requires. --- crates/accuracy/src/measured.rs | 163 ++++++++++++++++-- .../tests/measured_sketch_accuracy.rs | 17 +- crates/cost/src/lib.rs | 3 +- crates/cost/src/measured.rs | 120 +------------ crates/cost/src/model.rs | 2 +- crates/cost/tests/measured_operation_costs.rs | 1 - crates/devtools/src/rqe_atomic_costs.rs | 50 ++++-- crates/planner/src/lib.rs | 5 +- .../src/tests/edges.rs | 5 +- docs/develop_docs/library-api.md | 4 +- 10 files changed, 214 insertions(+), 156 deletions(-) diff --git a/crates/accuracy/src/measured.rs b/crates/accuracy/src/measured.rs index eac86804f..a3c66f156 100644 --- a/crates/accuracy/src/measured.rs +++ b/crates/accuracy/src/measured.rs @@ -9,11 +9,14 @@ //! request at 0.001 gets no measured guarantee, so the theoretical one //! applies. -use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr}; use asap_types::ir::schema::FieldDataType; +use asap_types::stages::ValueSign; use asap_types::summary_type::{SketchParams, SketchStatistic}; use serde::{Deserialize, Serialize}; +use crate::guarantee::guarantee_before_failure_probability_check; +use crate::readout::readout_error_metric; use crate::AccuracyEvidence; /// Errors measured on the executor's own sketches. Default: empty, every @@ -24,7 +27,13 @@ pub struct MeasuredSketchAccuracy { pub entries: Vec, } -/// One sketch configuration, read for one statistic, in one metric. +/// One sketch configuration, read for one statistic, in one metric. Each +/// recorded error is the worst case over the merge counts the calibration +/// measured: a state built over one window and a state merged from many +/// windows of the same θ share the entry, so its error must hold for +/// every one of them. Example: KLL k = 547 measured unmerged and merged +/// from 5 and 60 windows, with rank errors 0.0021, 0.0024 and 0.0023 at +/// δ = 0.01, records 0.0024 at 0.01. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MeasuredAccuracyEntry { @@ -64,6 +73,8 @@ pub enum MeasuredSketchAccuracyError { entry: usize, failure_probability: f64, }, + /// Entry `entry` records no point, so it can answer no request. + NoPoints { entry: usize }, /// Entry `entry` has a negative or non-finite error. InvalidError { entry: usize, error: f64 }, /// Entry `entry`'s failure probabilities are not ascending, or its @@ -75,23 +86,155 @@ pub enum MeasuredSketchAccuracyError { } impl MeasuredSketchAccuracy { - /// Check every entry: failure probabilities in (0, 1] and at least - /// 3 / trials, errors finite and non-negative, ascending failure + /// Check every entry: at least one point, failure probabilities in + /// (0, 1] and at least 3 / trials, errors finite and non-negative, ascending failure /// probability with non-increasing error, and no two entries for one /// (sketch, statistic, metric). pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> { - todo!("measured accuracy table: implement after the tests are approved") + for (index, entry) in self.entries.iter().enumerate() { + entry.validate(index)?; + if let Some(first) = self.entries[..index] + .iter() + .position(|earlier| earlier.measures_the_same_as(entry)) + { + return Err(MeasuredSketchAccuracyError::DuplicateEntry { + first, + second: index, + }); + } + } + Ok(()) + } + + /// The entry measuring `sketch` read for `statistic` in `metric`. + fn entry( + &self, + sketch: &SketchParams, + statistic: &SketchStatistic, + metric: AccuracyMetric, + ) -> Option<&MeasuredAccuracyEntry> { + self.entries.iter().find(|entry| { + entry.sketch == *sketch + && entry.statistic == *statistic + && entry.accuracy_metric == metric + }) } } +impl MeasuredAccuracyEntry { + /// Check this entry, the `index`th of its table. + fn validate(&self, index: usize) -> Result<(), MeasuredSketchAccuracyError> { + // Fewer than 3 failures in `trials` cannot estimate the (1 − δ) + // quantile: 1,000 trials record δ ≥ 0.003. + let smallest_failure_probability = 3.0 / f64::from(self.trials); + if self.error_at.is_empty() { + return Err(MeasuredSketchAccuracyError::NoPoints { entry: index }); + } + for point in &self.error_at { + let failure_probability = point.failure_probability; + if !(failure_probability > 0.0 + && failure_probability <= 1.0 + && failure_probability >= smallest_failure_probability) + { + return Err(MeasuredSketchAccuracyError::FailureProbabilityOutOfRange { + entry: index, + failure_probability, + }); + } + if !(point.error.is_finite() && point.error >= 0.0) { + return Err(MeasuredSketchAccuracyError::InvalidError { + entry: index, + error: point.error, + }); + } + } + let monotone = self.error_at.windows(2).all(|pair| { + pair[0].failure_probability < pair[1].failure_probability + && pair[0].error >= pair[1].error + }); + match monotone { + true => Ok(()), + false => Err(MeasuredSketchAccuracyError::NotMonotone { entry: index }), + } + } + + /// Whether `other` measures the same sketch, statistic and metric. + fn measures_the_same_as(&self, other: &MeasuredAccuracyEntry) -> bool { + self.sketch == other.sketch + && self.statistic == other.statistic + && self.accuracy_metric == other.accuracy_metric + } + + /// The point at the largest recorded failure probability at most + /// `failure_probability`; `None` below the smallest (never + /// extrapolated). Example: recorded at 0.003, 0.01 and 0.05, a request + /// at 0.02 gets the point at 0.01. + fn point_at_most(&self, failure_probability: f64) -> Option { + self.error_at + .iter() + .filter(|point| point.failure_probability <= failure_probability) + .max_by(|a, b| a.failure_probability.total_cmp(&b.failure_probability)) + .copied() + } +} + +/// Whether θ's theoretical guarantee for `statistic` can be deterministic +/// (failure probability zero) under the data it requires: then a +/// measurement never replaces it. Example: DDSketch's relative error α +/// over values of one sign; a KLL's rank error always has a failure +/// probability. +fn has_deterministic_guarantee( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, +) -> bool { + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + 1.0, + Some(ValueSign::NonNegative), + None, + ) + .is_some_and(|guarantee| guarantee.spec.failure_probability == ProbabilityExpr::Zero) +} + +/// Loaders must call [`MeasuredSketchAccuracy::validate`] before handing a +/// table to the planner: `measured_guarantee` does not re-check it, so an +/// unvalidated table could give Pass 2 non-monotone or empty entries. impl AccuracyEvidence for MeasuredSketchAccuracy { + /// The entry for exactly `parameters` and `statistic`, in the + /// readout's own metric (`Rank` for a KLL quantile), at its largest + /// recorded failure probability at most `failure_probability`; the + /// guarantee states that recorded failure probability. `None` when no + /// entry matches, below its smallest recorded failure probability, at + /// a failure probability that is not positive, and for a θ whose + /// theoretical guarantee can be deterministic (DDSketch). fn measured_guarantee( &self, - _summary_type: &FieldDataType, - _parameters: &SketchParams, - _statistic: &SketchStatistic, - _failure_probability: f64, + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, ) -> Option { - todo!("measured accuracy table: implement after the tests are approved") + if failure_probability.is_nan() || failure_probability <= 0.0 { + return None; + } + let metric = readout_error_metric(summary_type, statistic)?; + // Look the entry up first: the deterministic-guarantee check below + // computes a theoretical guarantee, so it runs only for a θ that + // has an entry. + let entry = self.entry(parameters, statistic, metric)?; + if has_deterministic_guarantee(summary_type, parameters, statistic) { + return None; + } + let point = entry.point_at_most(failure_probability)?; + Some(AccuracyGuarantee::new( + metric, + BoundExpr::Constant { value: point.error }, + ProbabilityExpr::Constant { + value: point.failure_probability, + }, + )) } } diff --git a/crates/accuracy/tests/measured_sketch_accuracy.rs b/crates/accuracy/tests/measured_sketch_accuracy.rs index 63b2a9a32..fb0632d33 100644 --- a/crates/accuracy/tests/measured_sketch_accuracy.rs +++ b/crates/accuracy/tests/measured_sketch_accuracy.rs @@ -91,8 +91,15 @@ fn lookup_no_extrapolation_below_smallest() { /// 0.004998 · 3.0902 / 2.5758 = 0.005997 at δ = 0.002. #[test] fn closed_form_applies_below_the_measured_range() { - let guarantee = - theoretical_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002).unwrap(); + let guarantee = theoretical_guarantee( + &kll(), + &SketchParams::Kll { k: 547 }, + &p99(), + 0.002, + None, + None, + ) + .unwrap(); let (error, _, metric) = parts(&guarantee); assert_eq!(metric, AccuracyMetric::Rank); assert!((error - 0.005997).abs() < 5e-6, "{error}"); @@ -205,7 +212,7 @@ fn table_validation() { ..kll_547_entry() }], }; - let rejected: [(&str, MeasuredSketchAccuracy, IsReason); 8] = [ + let rejected: [(&str, MeasuredSketchAccuracy, IsReason); 9] = [ // δ = 0.002 < 3 / 1,000 trials. ( "delta below 3/trials", @@ -217,6 +224,10 @@ fn table_validation() { } }, ), + // An entry with no points answers no request. + ("no points", with(vec![]), |e| { + *e == E::NoPoints { entry: 0 } + }), // The error grows with δ. ( "error increasing in delta", diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 0d60737b0..fa3e46a54 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -31,8 +31,7 @@ mod resources; pub(crate) use measured::aggregate_lookup; pub use measured::{ - lookup, CostTableProvenance, MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, - OperatorCost, OperatorKind, StateFamily, SummaryOperation, + lookup, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, StateFamily, }; pub use model::{ CalibrationFit, ClusterLimits, ClusterObjective, CostInputs, CostModel, ResourceSpeed, diff --git a/crates/cost/src/measured.rs b/crates/cost/src/measured.rs index 7a660ce44..6e6c22c1b 100644 --- a/crates/cost/src/measured.rs +++ b/crates/cost/src/measured.rs @@ -33,10 +33,6 @@ //! (operations × `ms_per_cpu_op`) is the measured time, and every cost model //! prices it as it prices any operation. //! -//! The table of earlier versions, a JSON array of [`MeasuredOperationCost`] -//! (one sketch operation each), still loads: each entry becomes the -//! [`OperatorCost`] of its operator at both phases. -//! //! [`CostInputs::measured_operation_costs`]: crate::CostInputs::measured_operation_costs //! [`ResourceUsage`]: crate::ResourceUsage //! [`ResourceSpeed`]: crate::ResourceSpeed @@ -53,32 +49,19 @@ use crate::node::analytical::AnalyticalCostError; pub use asap_types::ir::operator::OperatorKind; /// The deployment's measured cost table -/// ([`CostInputs::measured_operation_costs`]). In JSON, an object -/// with these fields, or (the format of earlier versions) an array of -/// [`MeasuredOperationCost`]. Default: empty, every node priced from the -/// formulas. +/// ([`CostInputs::measured_operation_costs`]). In JSON, an object with +/// these fields; an omitted field takes its default, an unknown one is +/// refused. Default: empty, every node priced from the formulas. /// /// [`CostInputs::measured_operation_costs`]: crate::CostInputs::measured_operation_costs -#[derive(Debug, Clone, Default, PartialEq, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] pub struct MeasuredOperationCosts { /// One entry per (operator, phase, family) the runtime measured. pub operators: Vec, /// CPU nanoseconds the runtime pays once per query request, whatever /// its plan: scheduling, request handling and result encoding. Default 0. pub ns_per_evaluation: f64, - /// Which executor and machine the numbers were measured with; the table - /// is re-measured when either changes. Recorded, not read. - pub provenance: Option, -} - -/// Where a measured table's numbers come from. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CostTableProvenance { - /// The executor build measured, e.g. a crate version or commit. - pub executor_version: String, - /// The machine it ran on, e.g. a CPU model or instance type. - pub machine: String, } /// The measured CPU time of one operator kind, at one phase, for one family. @@ -337,99 +320,6 @@ pub(crate) fn validate( Ok(()) } -// ── The format of earlier versions ────────────────────────────────────── - -/// One operation on a sketch's state, as an entry of the earlier format -/// ([`MeasuredOperationCost`]) names it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryOperation { - /// Insert one input row: a `SummaryAgg` or `SummaryTumblingWindowChain` entry. - Update, - /// Merge one input state into another: a `SummaryMerge` entry. - Merge, - /// Read one result out of one state: a `SummaryEstimate` entry. - Estimate, -} - -/// An entry of the earlier format, still loadable: the measured CPU time of -/// one operation on one sketch configuration. It becomes the -/// [`OperatorCost`] of its operator (both `SummaryAgg` and `SummaryTumblingWindowChain` for -/// an update) at both phases. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MeasuredOperationCost { - /// The sketch and its parameters, e.g. `{"Kll": {"k": 200}}`. - pub sketch: SketchParams, - /// Which operation was measured. - pub operation: SummaryOperation, - /// CPU nanoseconds one such operation takes on one core. - pub cpu_ns_per_operation: f64, -} - -impl From> for MeasuredOperationCosts { - fn from(entries: Vec) -> Self { - let operators = entries - .into_iter() - .flat_map(|entry| { - let kinds: &[OperatorKind] = match entry.operation { - SummaryOperation::Update => &[ - OperatorKind::SummaryAgg, - OperatorKind::SummaryTumblingWindowChain, - ], - SummaryOperation::Merge => &[OperatorKind::SummaryMerge], - SummaryOperation::Estimate => &[OperatorKind::SummaryEstimate], - }; - kinds - .iter() - .map(|&operator| OperatorCost { - operator, - phase: None, - sketch: Some(entry.sketch.clone()), - exact: None, - measure: None, - ns_per_unit: entry.cpu_ns_per_operation, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - }) - .collect::>() - }) - .collect(); - Self { - operators, - ..Self::default() - } - } -} - -/// Reads the table, or an array in the earlier format. -impl<'de> Deserialize<'de> for MeasuredOperationCosts { - fn deserialize>(deserializer: D) -> Result { - use serde::de::Error; - let value = serde_json::Value::deserialize(deserializer)?; - if value.is_array() { - return Vec::::deserialize(value) - .map(Self::from) - .map_err(D::Error::custom); - } - let fields = TableFields::deserialize(value).map_err(D::Error::custom)?; - Ok(Self { - operators: fields.operators, - ns_per_evaluation: fields.ns_per_evaluation, - provenance: fields.provenance, - }) - } -} - -/// The table's fields as JSON states them; omitted ones take their default. -#[derive(Default, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct TableFields { - operators: Vec, - ns_per_evaluation: f64, - provenance: Option, -} - /// Which costs one node's CPU operations came from. A node is one operator /// node of the candidate's physical plan DAG (`PhysicalASAPDAG`), keyed by /// its `PhysicalASAPNodeId` in [`PlanCost::per_node`]; not a machine. diff --git a/crates/cost/src/model.rs b/crates/cost/src/model.rs index 40e16c35c..9908f6a5c 100644 --- a/crates/cost/src/model.rs +++ b/crates/cost/src/model.rs @@ -240,7 +240,7 @@ pub struct CostInputs { /// The deployment's measured CPU time per unit of each operator; a node /// with an entry is priced from it, every other node from the formulas /// scaled by the calibration factor (module doc, "Measured operation - /// costs"). Also loads the earlier array of [`MeasuredOperationCost`](crate::MeasuredOperationCost). + /// costs"). /// Default empty: every node is priced from the formulas. pub measured_operation_costs: MeasuredOperationCosts, /// Externally established formula-to-measurement scale, held constant across diff --git a/crates/cost/tests/measured_operation_costs.rs b/crates/cost/tests/measured_operation_costs.rs index 4293ab13a..047fe473d 100644 --- a/crates/cost/tests/measured_operation_costs.rs +++ b/crates/cost/tests/measured_operation_costs.rs @@ -204,7 +204,6 @@ fn full_table() -> MeasuredOperationCosts { }, ], ns_per_evaluation: 50_000.0, - ..MeasuredOperationCosts::default() } } diff --git a/crates/devtools/src/rqe_atomic_costs.rs b/crates/devtools/src/rqe_atomic_costs.rs index 78f776577..bd7679804 100644 --- a/crates/devtools/src/rqe_atomic_costs.rs +++ b/crates/devtools/src/rqe_atomic_costs.rs @@ -28,7 +28,7 @@ //! //! [`CostInputs::measured_operation_costs`]: asap_cost::CostInputs::measured_operation_costs -use asap_cost::{MeasuredOperationCost, MeasuredOperationCosts, SummaryOperation}; +use asap_cost::{MeasuredOperationCosts, OperatorCost, OperatorKind}; use asap_types::summary_type::SketchParams; use serde_json::Value; @@ -61,10 +61,7 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result let rows = rows .as_array() .ok_or("rqe_atomic_costs.json is not a JSON array of rows")?; - // Test-first stub: the rows are still converted into the earlier - // format's entries, which the implementation replaces by `OperatorCost` - // entries. - let mut legacy_entries: Vec = Vec::new(); + let mut operators: Vec = Vec::new(); let mut skipped_rows = Vec::new(); for (index, row) in rows.iter().enumerate() { let sketch_name = row["sketch"] @@ -82,15 +79,24 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result } Err(error) => return Err(format!("row {index} ({sketch_name}): {error}")), }; - if legacy_entries.iter().any(|entry| entry.sketch == sketch) { + if operators + .iter() + .any(|entry| entry.sketch.as_ref() == Some(&sketch)) + { return Err(format!( "row {index} ({sketch_name}): a second row for {sketch:?}" )); } - for (field, operation) in [ - ("insert_cpu_secs", SummaryOperation::Update), - ("merge_cpu_secs", SummaryOperation::Merge), - ("query_cpu_secs", SummaryOperation::Estimate), + // An update prices both operators that fold rows into a state; a + // merge and an estimate their one operator. + let updates: &[OperatorKind] = &[ + OperatorKind::SummaryAgg, + OperatorKind::SummaryTumblingWindowChain, + ]; + for (field, kinds) in [ + ("insert_cpu_secs", updates), + ("merge_cpu_secs", &[OperatorKind::SummaryMerge]), + ("query_cpu_secs", &[OperatorKind::SummaryEstimate]), ] { let seconds = row[field] .as_f64() @@ -98,15 +104,25 @@ pub fn convert_rqe_atomic_costs(rows: &Value) -> Result .ok_or(format!( "row {index} ({sketch_name}): `{field}` is not a non-negative number" ))?; - legacy_entries.push(MeasuredOperationCost { - sketch: sketch.clone(), - operation, - cpu_ns_per_operation: seconds * 1e9, - }); + operators.extend(kinds.iter().map(|&operator| OperatorCost { + operator, + phase: None, + sketch: Some(sketch.clone()), + exact: None, + measure: None, + ns_per_unit: seconds * 1e9, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + })); } } - let _ = (legacy_entries, skipped_rows); - todo!("write the object form: OperatorCost entries, phase None") + Ok(ConvertedCosts { + measured_operation_costs: MeasuredOperationCosts { + operators, + ..MeasuredOperationCosts::default() + }, + skipped_rows, + }) } /// Why a row whose name has no planner sketch is skipped. diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index b7b0cf9ce..6002be097 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -60,9 +60,8 @@ pub use installable::{ // What a deployment runtime needs to build a `UserInput` and read an // `InstallablePlan`, so it depends on this crate and `asap-executor` alone. pub use asap_cost::{ - ClusterLimits, ClusterObjective, CostInputs, CostModel, CostTableProvenance, - MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, - ResourceSpeed, ResourceUnitCost, SummaryOperation, + ClusterLimits, ClusterObjective, CostInputs, CostModel, MeasuredOperationCosts, NodeCostSource, + OperatorCost, OperatorKind, ResourceSpeed, ResourceUnitCost, }; pub use asap_frontend_sql::SqlCatalog; pub use asap_stage3_plan_selection::{ diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs index b4ed7370d..2883c676b 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs @@ -145,14 +145,15 @@ fn measured_p99( } /// `quantile_over_time(0.99, m[5m])` over one series of `m` (so δ is not -/// split across readout values), with a per-value `metric` target of +/// split across readout values) whose values are non-negative (so +/// DDSketch's sign condition holds), with a per-value `metric` target of /// `max_error` at δ = 0.01: the memo, its state group and readout group. fn p99_with_per_value_target( algorithm: SketchAlgorithm, metric: AccuracyMetric, max_error: f64, ) -> (Memo, MemoGroupId, MemoGroupId) { - let mut memo = Memo::new(&with_series("m", 1)); + let mut memo = Memo::new(&with_non_negative_values("m").with_series_count(1)); let (_, state, readout) = quantile_over_time(&mut memo, algorithm, 0.99, 300); query( &mut memo, diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 39f98ece6..ccabd5320 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -194,10 +194,10 @@ asap_stage3_plan_selection::select_plan( | Input or output | Meaning | | --- | --- | -| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | +| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees, e.g. `MeasuredSketchAccuracy` loaded from a measured table and validated; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | | `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | | `DeploymentCapabilities` (`asap_types::deployment_capabilities`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries), `raw_data_retained` (default `true`; when `false`, stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`), `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains), `counter_semantics` (default: resets handled and extrapolated), `state_not_mergeable_across_windows` (default none) and `excluded_candidates` (`CandidateExclusion`s, default none). Every capability left out expands into exclusions (`DeploymentCapabilities::exclusions`, expanded once by the facade): Pass 1 skips the summaries they rule out outright, and Stage 2 applies them as one filter, pruning each matched candidate with `PruneReason::ExcludedByDeployment(source)`; with no plan left, Stage 3's rejection is `RejectionCause::ExcludedByDeployment` | -| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`, `raw_batch_width_ms`: the time one batch of raw rows covers in the data store, default 10,000, an assumed value), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | +| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`, `raw_batch_width_ms`: the time one batch of raw rows covers in the data store, default 10,000, an assumed value), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; an unknown field is refused). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory}` chooses among the candidates that fit | From 4fc18d04b709c9151b614055932f338bab805917 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 17 Nov 2026 06:40:10 +0000 Subject: [PATCH 3/3] fix(accuracy): MeasuredSketchAccuracy review fixes: one same-entry check; evidence scope and DDSketch exception in the docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MeasuredAccuracyEntry::measures is the one (sketch, statistic, metric) check, used by the lookup and the duplicate check. The docs state that the table is keyed per θ, independent of the population, holds the worst error over the measured merge counts, and that its coverage is the deployment's responsibility; that a deterministic guarantee (DDSketch) is never replaced by a measurement; and list measured.rs in the crate layout. --- crates/accuracy/src/measured.rs | 29 ++++++++++--------- docs/design_docs/concepts/accuracy-models.md | 12 ++++++-- .../asap-aware-mapping-architecture.md | 2 +- .../asap-aware-mapping-contracts.md | 3 +- docs/develop_docs/library-api.md | 2 +- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/crates/accuracy/src/measured.rs b/crates/accuracy/src/measured.rs index a3c66f156..673b53b50 100644 --- a/crates/accuracy/src/measured.rs +++ b/crates/accuracy/src/measured.rs @@ -93,10 +93,9 @@ impl MeasuredSketchAccuracy { pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> { for (index, entry) in self.entries.iter().enumerate() { entry.validate(index)?; - if let Some(first) = self.entries[..index] - .iter() - .position(|earlier| earlier.measures_the_same_as(entry)) - { + if let Some(first) = self.entries[..index].iter().position(|earlier| { + earlier.measures(&entry.sketch, &entry.statistic, entry.accuracy_metric) + }) { return Err(MeasuredSketchAccuracyError::DuplicateEntry { first, second: index, @@ -113,11 +112,9 @@ impl MeasuredSketchAccuracy { statistic: &SketchStatistic, metric: AccuracyMetric, ) -> Option<&MeasuredAccuracyEntry> { - self.entries.iter().find(|entry| { - entry.sketch == *sketch - && entry.statistic == *statistic - && entry.accuracy_metric == metric - }) + self.entries + .iter() + .find(|entry| entry.measures(sketch, statistic, metric)) } } @@ -158,11 +155,15 @@ impl MeasuredAccuracyEntry { } } - /// Whether `other` measures the same sketch, statistic and metric. - fn measures_the_same_as(&self, other: &MeasuredAccuracyEntry) -> bool { - self.sketch == other.sketch - && self.statistic == other.statistic - && self.accuracy_metric == other.accuracy_metric + /// Whether this entry measures `sketch` read for `statistic` in + /// `metric`: the one key a table holds at most one entry for. + fn measures( + &self, + sketch: &SketchParams, + statistic: &SketchStatistic, + metric: AccuracyMetric, + ) -> bool { + self.sketch == *sketch && self.statistic == *statistic && self.accuracy_metric == metric } /// The point at the largest recorded failure probability at most diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index c698b99cf..b0518694b 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -213,7 +213,9 @@ each operator's error contract (`ErrorContract`, gathered per memo state in `ErrorMetricsOfMemo`) and the comparison of a bound with a target (`within_limit`); the deployment's measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a theoretical -guarantee. Only a sketch readout introduces error. Every other operator +guarantee, except for a θ whose theoretical guarantee can be deterministic +(DDSketch's relative error α): a measurement never replaces a guarantee that +holds with failure probability zero. Only a sketch readout introduces error. Every other operator passes its inputs' errors on through a contract, and the contracts are conservative: an operator with no contract for a metric fails the edge of an input whose error is in that metric (`RootPruneReason::MetricRule`), so an @@ -471,7 +473,11 @@ cheapest readouts (cost 2) misses B's target, so it bounds nothing. `AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the claimed guarantee applies: sources, filters, grouping, evaluation windows and all merged tumbling windows. Evidence for a narrower population cannot silently certify -a wider one. Data-workload facts are declared: the planner has no clock, so the caller +a wider one. `MeasuredSketchAccuracy`, the measured table, is keyed per θ +(sketch parameters, statistic, metric) and does not depend on the population: +each recorded error is the worst over the merge counts the calibration +measured. That it covers the populations the deployment plans for is the +deployment's responsibility, not checked by the planner. Data-workload facts are declared: the planner has no clock, so the caller drops facts that no longer hold before planning. Source contracts are assertions that the source or deployment must establish @@ -564,6 +570,8 @@ The `asap_accuracy` crate separates these responsibilities: accuracy/src/ ├── lib.rs # Re-exports: the public interfaces ├── evidence.rs # AccuracyEvidence: the deployment's measured accuracy +├── measured.rs # MeasuredSketchAccuracy: the measured table per θ, +│ # statistic and metric; its validation and lookup ├── error_contract/ # ErrorContract: each operator's error contract per │ # metric ├── metric_rule/ # ErrorMetricsOfMemo: the contracts, allowed metrics and diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 4565186bb..6930cd895 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -80,7 +80,7 @@ same pipeline behind the `OptimizationPass` trait. | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state/mod.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | | Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R), and with each query's exact-only reads $a_{q,e}$ and (X) (`milp/`, `exact_only.rs`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | -| The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | +| The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `MeasuredSketchAccuracy`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `measured.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | | The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | | What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities/`; Stage 2 reads it as the one list of exclusions it expands into (`DeploymentCapabilities::exclusions`); Pass 1 reads the same list only to skip a summary whose sketch algorithm one exclusion rules out outright (a `summary_family`-only exclusion); Stage 3 reads none of it | | The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `models.rs` (`PlanningModels`) and `pass.rs` (the trait); the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 7c5ea1b00..482cb3eca 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -62,7 +62,8 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- - A summary's guarantee is the deployment's measurement (`AccuracyEvidence::measured_guarantee`) when there is one, else its - algorithm's error formula (`theoretical_guarantee`); Pass 2 compares it + algorithm's error formula (`theoretical_guarantee`); a θ whose formula can + be deterministic (DDSketch's relative error α) keeps its formula; Pass 2 compares it with each readout's `RequiredAccuracy` (`ErrorMetricsOfMemo::edge_meets_logical_requirements`). It fails closed: no rule means no guarantee, and the edge is not admissible. diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index ccabd5320..2a76a0edb 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -194,7 +194,7 @@ asap_stage3_plan_selection::select_plan( | Input or output | Meaning | | --- | --- | -| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees, e.g. `MeasuredSketchAccuracy` loaded from a measured table and validated; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | +| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees, e.g. `asap_accuracy::MeasuredSketchAccuracy` loaded from a measured table and validated; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | | `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | | `DeploymentCapabilities` (`asap_types::deployment_capabilities`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries), `raw_data_retained` (default `true`; when `false`, stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`), `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains), `counter_semantics` (default: resets handled and extrapolated), `state_not_mergeable_across_windows` (default none) and `excluded_candidates` (`CandidateExclusion`s, default none). Every capability left out expands into exclusions (`DeploymentCapabilities::exclusions`, expanded once by the facade): Pass 1 skips the summaries they rule out outright, and Stage 2 applies them as one filter, pruning each matched candidate with `PruneReason::ExcludedByDeployment(source)`; with no plan left, Stage 3's rejection is `RejectionCause::ExcludedByDeployment` | | `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`, `raw_batch_width_ms`: the time one batch of raw rows covers in the data store, default 10,000, an assumed value), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; an unknown field is refused). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which) |