diff --git a/Cargo.lock b/Cargo.lock index 84b535f2d..2c1d933af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,18 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "aqpbm-datagen" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=cf8f86735e6f4c59d9827e3ae5d5bd28e8cd3ee1#cf8f86735e6f4c59d9827e3ae5d5bd28e8cd3ee1" +dependencies = [ + "rand 0.9.5", + "rand_distr", + "rand_xoshiro", + "serde", + "thiserror 2.0.18", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -286,6 +298,7 @@ version = "0.1.0" dependencies = [ "asap-types", "asap_sketchlib", + "serde", "serde_json", ] @@ -305,6 +318,7 @@ dependencies = [ name = "asap-devtools" version = "0.1.0" dependencies = [ + "aqpbm-datagen", "asap-accuracy", "asap-cost", "asap-executor", @@ -2580,6 +2594,25 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "recursive" version = "0.1.1" diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 666477f7e..7d2da01b9 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } +serde = { version = "1", features = ["derive"] } serde_json = "1" [dev-dependencies] diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index abf154e7d..72638069e 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -2,7 +2,8 @@ //! theoretical guarantee of reading a statistic from a summary with given //! parameters ([`theoretical_guarantee`], from the per-algorithm //! guarantees in `guarantee`), the deployment's measured accuracy -//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract +//! ([`AccuracyEvidence`], in `evidence`, and the measured table +//! [`MeasuredSketchAccuracy`], in `measured`), each operator's error contract //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the //! metric of a readout's error from `readout`), each readout's required @@ -19,6 +20,7 @@ mod error_contract; mod evidence; mod guarantee; mod logical_edge; +mod measured; mod metric_rule; mod readout; mod readout_caps; @@ -27,5 +29,9 @@ pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::within_limit; +pub use measured::{ + ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy, + MeasuredSketchAccuracyError, +}; pub use metric_rule::ErrorMetricsOfMemo; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/measured.rs b/crates/accuracy/src/measured.rs new file mode 100644 index 000000000..6481dc64b --- /dev/null +++ b/crates/accuracy/src/measured.rs @@ -0,0 +1,97 @@ +//! The deployment's measured sketch accuracy: the errors a calibration run +//! measured on the executor's own sketches, which Pass 2 reads through +//! [`AccuracyEvidence`] before each summary's theoretical guarantee. +//! +//! Example: KLL with `k = 800`, read for `Quantile { q: 0.99 }`, measured +//! over 1,000 trials, has rank error 0.00176 at failure probability 0.01 +//! and 0.00121 at 0.05. A request at failure probability 0.02 gets +//! 0.00176 (the largest recorded failure probability at most 0.02); a +//! request at 0.001 gets no measured guarantee, so the theoretical one +//! applies. + +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchParams, SketchStatistic}; +use serde::{Deserialize, Serialize}; + +use crate::AccuracyEvidence; + +/// Errors measured on the executor's own sketches. Default: empty, every +/// sketch keeps its theoretical guarantee. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredSketchAccuracy { + pub entries: Vec, +} + +/// One sketch configuration, read for one statistic, in one metric. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredAccuracyEntry { + /// The exact parameters measured; only these match. + pub sketch: SketchParams, + /// The readout, e.g. `Quantile { q: 0.99 }`; only the same statistic + /// matches. + pub statistic: SketchStatistic, + /// The metric every recorded error is in, e.g. `Rank` for KLL. + pub accuracy_metric: AccuracyMetric, + /// Independent trials; every recorded failure probability is at least + /// 3 / trials. + pub trials: u32, + /// Ascending failure probability; the error does not increase as the + /// failure probability grows. + pub error_at: Vec, +} + +/// One measured point: the error not exceeded except with probability +/// `failure_probability`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErrorAtFailureProbability { + pub failure_probability: f64, + /// The (1 − δ) quantile of the per-value errors, the worst over the + /// measured merge counts, in the entry's `accuracy_metric`. + pub error: f64, +} + +/// Why a measured accuracy table is not valid. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum MeasuredSketchAccuracyError { + /// Entry `entry` has a failure probability outside (0, 1] or below + /// 3 / trials. + FailureProbabilityOutOfRange { + entry: usize, + failure_probability: f64, + }, + /// Entry `entry` has a negative or non-finite error. + InvalidError { entry: usize, error: f64 }, + /// Entry `entry`'s failure probabilities are not ascending, or its + /// error increases as the failure probability grows. + NotMonotone { entry: usize }, + /// Entries `first` and `second` measure the same sketch, statistic and + /// metric. + DuplicateEntry { first: usize, second: usize }, +} + +impl MeasuredSketchAccuracy { + /// Check every entry: failure probabilities in (0, 1] and at least + /// 3 / trials, errors finite and non-negative, ascending failure + /// probability with non-increasing error, and no two entries for one + /// (sketch, statistic, metric). + pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> { + todo!("asap_run: implement after test approval") + } +} + +impl AccuracyEvidence for MeasuredSketchAccuracy { + fn measured_guarantee( + &self, + _summary_type: &FieldDataType, + _parameters: &SketchParams, + _statistic: &SketchStatistic, + _failure_probability: f64, + ) -> Option { + todo!("asap_run: implement after test approval") + } +} diff --git a/crates/accuracy/tests/measured_sketch_accuracy.rs b/crates/accuracy/tests/measured_sketch_accuracy.rs new file mode 100644 index 000000000..34eeccf9f --- /dev/null +++ b/crates/accuracy/tests/measured_sketch_accuracy.rs @@ -0,0 +1,245 @@ +//! The measured accuracy table as Pass 2's [`AccuracyEvidence`]: a request +//! at failure probability δ takes the largest recorded δ' ≤ δ of an entry +//! with exactly its parameters, statistic and metric, and nothing else. +//! Every expected value is read off the fixture by hand. + +use asap_accuracy::{ + theoretical_guarantee, AccuracyEvidence, ErrorAtFailureProbability, MeasuredAccuracyEntry, + MeasuredSketchAccuracy, +}; +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; + +fn kll() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +fn p99() -> SketchStatistic { + SketchStatistic::Quantile { q: 0.99 } +} + +fn point(failure_probability: f64, error: f64) -> ErrorAtFailureProbability { + ErrorAtFailureProbability { + failure_probability, + error, + } +} + +/// KLL k = 547 read for the 0.99 quantile, rank error over 1,000 trials: +/// 0.0031 at δ = 0.003, 0.0024 at 0.01, 0.0016 at 0.05. +fn kll_547_entry() -> MeasuredAccuracyEntry { + MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k: 547 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Rank, + trials: 1_000, + error_at: vec![ + point(0.003, 0.0031), + point(0.01, 0.0024), + point(0.05, 0.0016), + ], + } +} + +fn table() -> MeasuredSketchAccuracy { + MeasuredSketchAccuracy { + entries: vec![kll_547_entry()], + } +} + +/// The error, failure probability and metric of a guarantee. +fn parts(guarantee: &AccuracyGuarantee) -> (f64, f64, AccuracyMetric) { + ( + guarantee.spec.error.evaluate().unwrap(), + guarantee.spec.failure_probability.evaluate().unwrap(), + guarantee.spec.accuracy_metric, + ) +} + +/// δ = 0.01 takes 0.01's error; 0.02 takes 0.01's too (the largest +/// recorded δ' ≤ 0.02, not the nearest 0.05 above); 0.05 and 1.0 (no +/// requirement) take 0.05's. +#[test] +fn lookup_takes_largest_recorded_delta_at_most_request() { + let table = table(); + let at = |delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + }; + assert_eq!(at(0.01), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.02), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.05), (0.0016, 0.05, AccuracyMetric::Rank)); + assert_eq!(at(1.0), (0.0016, 0.05, AccuracyMetric::Rank)); +} + +/// δ = 0.002 is below every recorded δ': no measured guarantee, never +/// 0.0031 extrapolated downward. +#[test] +fn lookup_no_extrapolation_below_smallest() { + let guarantee = + table().measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002); + assert_eq!(guarantee, None); +} + +/// Where the table has no measurement (δ = 0.002 above), the theoretical +/// guarantee applies: KLL k = 547 has rank error 2.296 / 547^0.9723 = +/// 0.004998 at δ ≥ 0.01, scaled below 0.01 by z(1 − δ/2) / z(0.995): +/// 0.004998 · 3.0902 / 2.5758 = 0.005997 at δ = 0.002. +#[test] +fn closed_form_applies_below_the_measured_range() { + let guarantee = + theoretical_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002).unwrap(); + let (error, _, metric) = parts(&guarantee); + assert_eq!(metric, AccuracyMetric::Rank); + assert!((error - 0.005997).abs() < 5e-6, "{error}"); +} + +/// Only the exact parameters, statistic and metric match: k = 548, the +/// median, or the relative metric get nothing. +#[test] +fn lookup_exact_params_statistic_metric() { + let table = table(); + assert_eq!( + table.measured_guarantee(&kll(), &SketchParams::Kll { k: 548 }, &p99(), 0.01), + None + ); + assert_eq!( + table.measured_guarantee( + &kll(), + &SketchParams::Kll { k: 547 }, + &SketchStatistic::Quantile { q: 0.5 }, + 0.01 + ), + None + ); + let relative = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + accuracy_metric: AccuracyMetric::Relative, + ..kll_547_entry() + }], + }; + // provisional answer 7: an entry in another metric than the readout's + // own (Rank for KLL) is never returned. + assert_eq!( + relative.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.01), + None + ); +} + +/// An empty table measures nothing. +#[test] +fn empty_table_measures_nothing() { + let empty = MeasuredSketchAccuracy::default(); + for delta in [0.001, 0.01, 1.0] { + assert_eq!( + empty.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta), + None + ); + } +} + +/// Lookup monotonicity (I10): a larger δ never gets a larger measured +/// error than a smaller one. +#[test] +fn lookup_error_does_not_grow_with_delta() { + let table = table(); + let deltas = [0.003, 0.005, 0.01, 0.02, 0.05, 0.5, 1.0]; + let errors: Vec = deltas + .iter() + .map(|&delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + .0 + }) + .collect(); + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{errors:?}" + ); +} + +/// A deterministic sketch keeps its deterministic guarantee: a measured +/// (probabilistic) entry for DDSketch α = 0.01 is not returned. +// provisional answer 8 +#[test] +fn measured_entry_never_replaces_a_deterministic_guarantee() { + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: SketchParams::DDSketch { alpha: 0.01 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.004)], + }], + }; + let guarantee = table.measured_guarantee( + &FieldDataType::Sketch(SketchAlgorithm::DDSketch), + &SketchParams::DDSketch { alpha: 0.01 }, + &p99(), + 0.01, + ); + assert_eq!(guarantee, None); +} + +/// Each invalid table is rejected. +#[test] +fn table_validation() { + assert!(table().validate().is_ok()); + let with = |error_at: Vec| MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + error_at, + ..kll_547_entry() + }], + }; + let rejected = [ + // δ = 0.002 < 3 / 1,000 trials. + ("delta below 3/trials", with(vec![point(0.002, 0.004)])), + // The error grows with δ. + ( + "error increasing in delta", + with(vec![point(0.01, 0.002), point(0.05, 0.003)]), + ), + // δ not ascending. + ( + "unsorted deltas", + with(vec![point(0.05, 0.0016), point(0.01, 0.0024)]), + ), + ("delta zero", with(vec![point(0.0, 0.01)])), + ("delta above one", with(vec![point(1.5, 0.01)])), + ("negative error", with(vec![point(0.01, -1.0)])), + ("NaN error", with(vec![point(0.01, f64::NAN)])), + ( + "duplicate entry", + MeasuredSketchAccuracy { + entries: vec![kll_547_entry(), kll_547_entry()], + }, + ), + ]; + for (name, table) in rejected { + assert!(table.validate().is_err(), "{name} must be rejected"); + } +} + +/// The table's JSON is the issue's example format, and round-trips. +#[test] +fn accuracy_json_round_trips() { + let text = r#"{"entries": [{"sketch": {"Kll": {"k": 800}}, "statistic": {"Quantile": {"q": 0.99}}, + "accuracy_metric": "rank", "trials": 1000, "error_at": [ + {"failure_probability": 0.01, "error": 0.00176}, + {"failure_probability": 0.05, "error": 0.00121}]}]}"#; + let table: MeasuredSketchAccuracy = serde_json::from_str(text).unwrap(); + assert_eq!(table.entries[0].sketch, SketchParams::Kll { k: 800 }); + assert_eq!(table.entries[0].error_at[1], point(0.05, 0.00121)); + let written = serde_json::to_string(&table).unwrap(); + assert_eq!( + serde_json::from_str::(&written).unwrap(), + table + ); +} diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index efd664bf0..99bee20bf 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -2,6 +2,7 @@ use std::time::Duration; +use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::ExecutionDataState; use asap_types::ir::scalar::ColumnRef; use asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode; @@ -735,3 +736,146 @@ fn a_ddsketch_over_a_counter_declaring_its_value_unknown_keeps_both_stores() { ddsketch_costed_bytes(true, None) ); } + +/// A query-time producer of `rows` plain rows, for a node under test. +fn rows_of<'a>(producer: &'a PhysicalASAPDAGNode, rows: u64) -> ProducerOutput<'a> { + ProducerOutput { + payload: &producer.payload, + output_schema: &producer.output_schema, + output: EdgeStatistics { + rows, + bytes: rows * 16, + }, + sources: BTreeSet::new(), + family: StateFamily::None, + unchanged_columns: BTreeSet::new(), + tumbling_windows: 1, + chain_tumbling_window_width_ms: None, + } +} + +/// A one-minute `TimeRange` over node 0. +fn one_minute_range() -> PhysicalASAPDAGNode { + node(Payload::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: 0, + })) +} + +/// A query-time `SummaryAgg` of an exact sum over 600 input rows, with a +/// measured entry of 10 ns per row and 500 ns per computation, takes +/// 600 · 10 + 500 = 6,500 ns = 0.0065 ms per execution; computed once a +/// second, 6.5e-6 core-seconds per second, from the table. +#[test] +fn node_priced_by_hand() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::SummaryAgg, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 10.0, + ns_per_invocation: 500.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let build = node(Payload::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })); + let usage = node_resource_usage_and_cost( + &build, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.0065).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); + assert!( + (usage.cost.cpu_core_seconds_per_second - 6.5e-6).abs() < 1e-15, + "{}", + usage.cost.cpu_core_seconds_per_second + ); +} + +/// An `Aggregate` is priced from the table only when each of its measures +/// has an entry: with an entry for `sum` (10 ns per row) and none for +/// `quantile`, `Aggregate[sum, quantile]` keeps the formulas, and +/// `Aggregate[sum]` over 600 rows takes 600 · 10 = 6,000 ns. +#[test] +fn aggregate_needs_every_measure() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("sum".into()), + ns_per_unit: 10.0, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let aggregate = |measures: Vec| { + node(Payload::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::none()), + output_names: vec![String::new(); measures.len()], + filters: vec![None; measures.len()], + measures, + having: None, + child: 0, + })) + }; + let both = aggregate(vec![ + AggIntent::Sum { col: None }, + AggIntent::Quantile { col: None, q: 0.5 }, + ]); + let usage = node_resource_usage_and_cost( + &both, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Analytical); + let sum = aggregate(vec![AggIntent::Sum { col: None }]); + let usage = node_resource_usage_and_cost( + &sum, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.006).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); +} diff --git a/crates/cost/tests/measured_operation_costs.rs b/crates/cost/tests/measured_operation_costs.rs new file mode 100644 index 000000000..2a91ddd93 --- /dev/null +++ b/crates/cost/tests/measured_operation_costs.rs @@ -0,0 +1,245 @@ +//! The measured cost table: [`lookup`] (exact match, interpolation in log +//! size, phase shadowing, no extrapolation), its JSON round trip and the +//! formats the `asap_run` design deletes. Every expected price is +//! computed by hand from the entries below. + +use asap_cost::{ + lookup, CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind, StateFamily, +}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::summary_type::{ExactKind, SketchParams}; + +/// A sketch's size for interpolation: KLL's `k`, 1 for any other sketch. +fn size_of(params: &SketchParams) -> f64 { + match params { + SketchParams::Kll { k } => *k as f64, + _ => 1.0, + } +} + +/// A `SummaryAgg` entry for KLL with `k` at `phase`. +fn kll_entry( + k: u32, + phase: Option, + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +) -> OperatorCost { + OperatorCost { + operator: OperatorKind::SummaryAgg, + phase, + sketch: Some(SketchParams::Kll { k }), + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation, + memory_bytes_per_state, + } +} + +/// KLL `k = 200` at 100 ns/unit, 1,000 ns/invocation, 2,000 bytes, and +/// `k = 800` at 300 ns/unit, 3,000 ns/invocation, 8,000 bytes, for both +/// phases. +fn kll_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, Some(8_000.0)), + ], + ..MeasuredOperationCosts::default() + } +} + +/// A price's fields: ns per unit, ns per invocation, bytes per state. +#[derive(Debug, PartialEq)] +struct Price { + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +} + +fn kll_price_at(table: &MeasuredOperationCosts, phase: ExecutionTiming, k: u32) -> Option { + lookup( + table, + OperatorKind::SummaryAgg, + phase, + StateFamily::Sketch(&SketchParams::Kll { k }), + size_of, + ) + .map(|price| Price { + ns_per_unit: price.ns_per_unit, + ns_per_invocation: price.ns_per_invocation, + memory_bytes_per_state: price.memory_bytes_per_state, + }) +} + +fn assert_close(actual: f64, expected: f64) { + assert!( + (actual - expected).abs() <= 1e-9 * expected.abs().max(1.0), + "{actual} != {expected}" + ); +} + +/// An entry with exactly the node's parameters is its price, not an +/// interpolation. +#[test] +fn lookup_exact_params_win() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 200).unwrap(); + assert_eq!(price.ns_per_unit, 100.0); + assert_eq!(price.ns_per_invocation, 1_000.0); + assert_eq!(price.memory_bytes_per_state, Some(2_000.0)); +} + +/// k = 400 lies at ln(400/200) / ln(800/200) = 0.5 between the two +/// entries in log size: 100 + 0.5·200 = 200 ns/unit, 2,000 ns/invocation, +/// 5,000 bytes. Linear in size would give 166.7 ns/unit. +#[test] +fn lookup_interpolates_in_log_size() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 400).unwrap(); + assert_close(price.ns_per_unit, 200.0); + assert_close(price.ns_per_invocation, 2_000.0); + assert_close(price.memory_bytes_per_state.unwrap(), 5_000.0); +} + +/// Below the smallest and above the largest measured size, the nearest +/// entry's price: 100 at k = 100 (not 33), 300 at k = 1600 (not 367). +#[test] +fn lookup_no_extrapolation() { + let table = kll_table(); + let below = kll_price_at(&table, ExecutionTiming::QueryTime, 100).unwrap(); + let above = kll_price_at(&table, ExecutionTiming::QueryTime, 1_600).unwrap(); + assert_eq!(below.ns_per_unit, 100.0); + assert_eq!(above.ns_per_unit, 300.0); +} + +/// A query-time entry for k = 200 at 999 ns/unit hides both both-phase +/// entries at query time: k = 400 has only k = 200 to go by, so 999. At +/// ingestion time there is no ingestion entry, so the both-phase pair +/// interpolates to 200. +#[test] +fn lookup_phase_entries_shadow_both() { + let mut table = kll_table(); + table.operators.push(kll_entry( + 200, + Some(ExecutionTiming::QueryTime), + 999.0, + 0.0, + None, + )); + let query = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + let ingestion = kll_price_at(&table, ExecutionTiming::IngestionTime, 400).unwrap(); + assert_eq!(query.ns_per_unit, 999.0); + assert_close(ingestion.ns_per_unit, 200.0); +} + +/// With no bytes on k = 800, k = 400 has one memory neighbour only: no +/// measured bytes (the formulas' bytes apply), never an interpolation +/// with 0. +#[test] +fn lookup_memory_needs_both_neighbours() { + let table = MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, None), + ], + ..MeasuredOperationCosts::default() + }; + let price = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + assert_eq!(price.memory_bytes_per_state, None); +} + +/// KLL entries do not price a DDSketch node. +#[test] +fn lookup_other_algorithm_misses() { + let price = lookup( + &kll_table(), + OperatorKind::SummaryAgg, + ExecutionTiming::QueryTime, + StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), + size_of, + ); + assert!(price.is_none()); +} + +/// A table with one of everything a JSON file can hold. +fn full_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry( + 200, + Some(ExecutionTiming::IngestionTime), + 100.5, + 1_000.25, + Some(2_048.0), + ), + OperatorCost { + operator: OperatorKind::SummaryMerge, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 3.75, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("quantile".into()), + ns_per_unit: 12.0, + ns_per_invocation: 250.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: 20.0, + ns_per_invocation: 5_000.0, + memory_bytes_per_state: None, + }, + ], + ns_per_evaluation: 50_000.0, + ..MeasuredOperationCosts::default() + } +} + +/// `costs.json` round-trips: written, read back and written again it is +/// byte-equal, and it validates. +#[test] +fn costs_json_round_trips() { + let table = full_table(); + let written = serde_json::to_string_pretty(&table).unwrap(); + let read: MeasuredOperationCosts = serde_json::from_str(&written).unwrap(); + assert_eq!(read, table); + assert_eq!(serde_json::to_string_pretty(&read).unwrap(), written); + let inputs = CostInputs { + measured_operation_costs: read, + ..CostInputs::default() + }; + inputs.validate().unwrap(); +} + +/// The design deletes `provenance` (machine information lives in +/// `machine.json`) and the array format of earlier versions: a table +/// carrying either no longer loads. +// provisional answer 22 +#[test] +fn costs_json_rejects_removed_fields() { + let with_provenance = r#"{"operators": [], "ns_per_evaluation": 0.0, + "provenance": {"executor_version": "0.1.0", "machine": "x86_64"}}"#; + assert!( + serde_json::from_str::(with_provenance).is_err(), + "a table with `provenance` must not load" + ); + let legacy_array = r#"[{"sketch": {"Kll": {"k": 200}}, "operation": "update", + "cpu_ns_per_operation": 100.0}]"#; + assert!( + serde_json::from_str::(legacy_array).is_err(), + "a table in the array format must not load" + ); +} diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 80079502b..321b083d1 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -33,6 +33,10 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } +# The data `asap_run` ingests: sketch-bench's synthetic time-series +# generator, pinned to the commit that makes it a standalone library. Only +# the developer tools depend on it. +aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "cf8f86735e6f4c59d9827e3ae5d5bd28e8cd3ee1" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/devtools/src/bin/asap_run.rs b/crates/devtools/src/bin/asap_run.rs new file mode 100644 index 000000000..8057c742d --- /dev/null +++ b/crates/devtools/src/bin/asap_run.rs @@ -0,0 +1,20 @@ +//! `asap_run`: execute plans on the Planner's executor, calibrate measured +//! unit costs and sketch accuracy per machine, bench predicted against +//! measured cost, and compute source facts from data +//! (`asap_devtools::measurement`). +//! +//! ```text +//! asap_run execute --workload w.json [--data facts|file.jsonl] --at [,...] +//! asap_run calibrate [--families all|] [--small] +//! asap_run bench --workload w.json --arms exact,planned,noraw --duration +//! asap_run bench --suite showcase [--small] [--no-cost-gate] +//! asap_run facts --data --workload w.json [--trust-observed-bounds] +//! ``` +//! +//! Every subcommand reads the machine's tables as `asap-planner` does: +//! `--costs` / `--accuracy`, else `ASAP_CALIBRATION_DIR` or +//! `/calibration`, else the formulas with a warning. + +fn main() { + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 2b1e21f71..fcb859793 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -18,11 +18,13 @@ //! candidates for the viewer document. [`corpora`] splits the checked-in //! query corpora and holds their SQL catalogs. [`rqe_atomic_costs`] converts //! sketch-bench's measured per-operation costs into plan selection's -//! measured table. [`planner_fuzz`] is the metamorphic fuzz tester for +//! measured table. [`measurement`] runs selected plans on the executor +//! and measures them, behind the `asap_run` bin. [`planner_fuzz`] is the metamorphic fuzz tester for //! missed reuse and sharing behind the `planner_fuzz` bin. pub mod candidate_labels; pub mod corpora; +pub mod measurement; pub mod planner_fuzz; pub mod planner_layering_examples; pub mod planning_stages; diff --git a/crates/devtools/src/measurement/accuracy_calibration.rs b/crates/devtools/src/measurement/accuracy_calibration.rs new file mode 100644 index 000000000..e13b7cc01 --- /dev/null +++ b/crates/devtools/src/measurement/accuracy_calibration.rs @@ -0,0 +1,39 @@ +//! Accuracy calibration (`asap_run calibrate`): for each sketch +//! configuration × readout statistic × metric, seeded independent trials on +//! the executor's own sketches at each measured merge count, recorded as +//! the (1 − δ) quantile of the per-value errors, the worst over the merge +//! counts (`MeasuredSketchAccuracy`). +//! +//! Example: KLL `k = 547` read for `Quantile { q: 0.99 }` has rank errors +//! 0.0015, 0.0018 and 0.0017 at δ = 0.01 after 1, 6 and 360 merges; the +//! table records 0.0018. + +use asap_accuracy::{ErrorAtFailureProbability, MeasuredAccuracyEntry}; +use asap_types::accuracy::AccuracyMetric; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +/// The nearest-rank (1 − δ) quantile of `errors`: the ⌈(1 − δ)·n⌉-th +/// smallest. +pub fn error_quantile(errors: &[f64], failure_probability: f64) -> f64 { + let _ = (errors, failure_probability); + todo!("asap_run: implement after test approval") +} + +/// One configuration's errors after `merge_count` merges, as +/// `calibrate-run.json` records them. +#[derive(Debug, Clone, PartialEq)] +pub struct MergeCountErrors { + pub sketch: SketchParams, + pub statistic: SketchStatistic, + pub accuracy_metric: AccuracyMetric, + pub merge_count: u32, + pub trials: u32, + pub error_at: Vec, +} + +/// One table entry per (sketch, statistic, metric) of `results`: at each +/// failure probability, the worst error over the merge counts. +pub fn worst_over_merge_counts(results: &[MergeCountErrors]) -> Vec { + let _ = results; + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/benchmark.rs b/crates/devtools/src/measurement/benchmark.rs new file mode 100644 index 000000000..db2589c0d --- /dev/null +++ b/crates/devtools/src/measurement/benchmark.rs @@ -0,0 +1,215 @@ +//! Benchmarks (`asap_run bench`): each arm of a workload run end to end +//! over the same data, its measured usage and error beside the planner's +//! predicted cost, and the showcase gate. +//! +//! Gate example: an exact arm measured at cost 100 per second and a +//! selected plan at 105 pass (105 ≤ 1.05 × 100); 105.01 fails. A selected +//! plan whose measured error exceeds its target fails whatever its cost. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use asap_planner::{InstallablePlan, PlanningWorkload, SelectedPlanReport}; +use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; +use asap_types::workload::QueryWorkloadEntry; + +use crate::measurement::calibration_tables::PlanningTables; + +/// The plans one benchmark compares over the same data. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum BenchArm { + /// The exact plan: the reference every error is measured against. + Exact, + /// The plan selected with the resolved tables. + Planned, + /// The plan selected with `raw_data_retained: false`. + NoRaw, +} + +/// The data setting of one gate row, e.g. 1 Hz evaluated every 10 s with +/// raw data retained. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DataSetting { + pub sample_interval_ms: u64, + pub evaluation_interval_ms: u64, + pub raw_data_retained: bool, +} + +/// One `asap_run bench` run. +#[derive(Debug, Clone)] +pub struct BenchRequest<'a> { + pub workload: &'a PlanningWorkload, + pub arms: Vec, + pub settings: Vec, + pub series_count: u64, + pub duration_ms: u64, + /// Runs per arm; each measurement is their median. + pub repetitions: u32, + pub tables: PlanningTables, + pub seed: u64, + /// Whether the gate checks cost (`--no-cost-gate` turns it off; error + /// is always checked). + pub cost_gate: bool, + /// Where `runs///` goes; `None` writes nothing. + pub output_root: Option, +} + +/// What a run measured of one arm, per second of wall time unless the +/// name says otherwise. +#[derive(Debug, Clone, PartialEq)] +pub struct MeasuredUsage { + pub p50_query_ms: f64, + pub p95_query_ms: f64, + pub ingestion_cpu_seconds_per_second: f64, + pub query_cpu_seconds_per_second: f64, + pub average_memory_bytes: f64, + pub peak_memory_bytes: u64, + pub scanned_bytes_per_second: f64, +} + +/// One arm of one setting. +#[derive(Debug, Clone)] +pub struct ArmMeasurement { + pub arm: BenchArm, + pub setting: DataSetting, + pub plan: InstallablePlan, + /// The planner's prediction for this plan, with the same cost inputs. + pub predicted: SelectedPlanReport, + /// `Err` when the arm did not run. + pub measured: Result, + /// Cost model 1 over the measured usage at the planner's own prices: + /// the median of `repetition_costs_per_second`. + pub measured_cost_per_second: Option, + /// The same, per repetition, for the noise band. + pub repetition_costs_per_second: Vec, + /// The arm's error against the exact arm in the target's metric, at + /// the target's failure probability. + pub measured_error: Option, +} + +/// One gate row: one workload at one setting. +#[derive(Debug, Clone, PartialEq)] +pub struct GateRow { + pub workload: String, + pub setting: DataSetting, + /// `None` when the exact arm did not run. + pub exact_cost_per_second: Option, + /// `Err` when the selected arm did not run. + pub selected_cost_per_second: Result, + pub measured_error: f64, + pub target_error: f64, +} + +/// Why a gate failed. +#[derive(Debug, Clone, PartialEq)] +pub enum GateFailure { + /// The selected plan cost more than 1.05 × the exact plan. + CostAboveLimit { + workload: String, + setting: DataSetting, + }, + /// Its error exceeded the target. + ErrorAboveTarget { + workload: String, + setting: DataSetting, + }, + /// Its error is not a number. + ErrorNotANumber { + workload: String, + setting: DataSetting, + }, + ExactArmMissing { + workload: String, + setting: DataSetting, + }, + SelectedArmFailed { + workload: String, + setting: DataSetting, + }, + /// An expected row is absent. + MissingRow { + workload: String, + setting: DataSetting, + }, + /// A row nobody expected. + ExtraRow { + workload: String, + setting: DataSetting, + }, +} + +/// The gate's verdict over a run's rows. +#[derive(Debug, Clone, PartialEq)] +pub enum GateVerdict { + Pass, + Fail(Vec), +} + +/// The gate over `rows`, which must be exactly the `expected` (workload, +/// setting) pairs; `cost_gate` false checks error and completeness only. +pub fn gate_verdict( + expected: &[(String, DataSetting)], + rows: &[GateRow], + cost_gate: bool, +) -> GateVerdict { + let _ = (expected, rows, cost_gate); + todo!("asap_run: implement after test approval") +} + +/// What one benchmark run produced. +#[derive(Debug, Clone)] +pub struct BenchReport { + pub arms: Vec, + pub rows: Vec, + pub verdict: GateVerdict, + /// `runs///`, when written. + pub run_dir: Option, +} + +/// Run `request`. +pub fn bench(request: &BenchRequest<'_>) -> Result { + let _ = request; + todo!("asap_run: implement after test approval") +} + +/// The fixed showcase suite (`--suite showcase`), at the per-PR scale +/// when `small` (100 series, 10 minutes of data). +pub fn showcase_suite(small: bool) -> Vec<(String, PlanningWorkload, Vec)> { + let _ = small; + todo!("asap_run: implement after test approval") +} + +/// Run the showcase suite with `tables`. +pub fn bench_showcase( + small: bool, + tables: &PlanningTables, + output_root: Option, +) -> Result { + let _ = (small, tables, output_root); + todo!("asap_run: implement after test approval") +} + +/// Measure one given plan of `workload` at `setting` (the exhaustive +/// oracle runs every enumerated plan through it). +pub fn bench_plan( + plan: &InstallablePlan, + workload: &PlanningWorkload, + setting: &DataSetting, + request: &BenchRequest<'_>, +) -> Result { + let _ = (plan, workload, setting, request); + todo!("asap_run: implement after test approval") +} + +/// The installable plan of one plan over Stage 2's candidates: the +/// `candidates` chosen and the `edges` feeding each of their inputs (the +/// exhaustive oracle builds every enumerated plan with it). +pub fn installable_plan_of_selection( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + candidates: &BTreeSet, + edges: &BTreeSet, +) -> Result { + let _ = (physical, entries, candidates, edges); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/calibration_tables.rs b/crates/devtools/src/measurement/calibration_tables.rs new file mode 100644 index 000000000..c61423bb4 --- /dev/null +++ b/crates/devtools/src/measurement/calibration_tables.rs @@ -0,0 +1,130 @@ +//! A machine's calibration tables: running a calibration and writing its +//! directory, and finding and loading the tables a planning run reads +//! (shared by `asap_run` and `asap-planner`). +//! +//! Layout, under `ASAP_CALIBRATION_DIR` or `/calibration`: +//! `/{machine.json, costs.json, accuracy.json, +//! calibrate-run.json}`. Example: with `ASAP_CALIBRATION_DIR=/data/cal` on +//! machine `x86_64-intel-xeon-gold-6142-56c-registered-ddr4`, the cost table is +//! `/data/cal/x86_64-intel-xeon-gold-6142-56c-registered-ddr4/costs.json`, unless `--costs ` +//! names another. + +use std::path::{Path, PathBuf}; + +use asap_accuracy::MeasuredSketchAccuracy; +use asap_cost::{CostInputs, MeasuredOperationCosts}; + +use crate::measurement::accuracy_calibration::MergeCountErrors; +use crate::measurement::cost_calibration::{CalibrationFamilies, CostFitRecord}; +use crate::measurement::machine::MachineDescription; + +/// The environment variable naming the calibration directory. +pub const CALIBRATION_DIR_ENV: &str = "ASAP_CALIBRATION_DIR"; + +/// One `asap_run calibrate` run. +#[derive(Debug, Clone, PartialEq)] +pub struct CalibrateRequest { + pub families: CalibrationFamilies, + /// The per-PR scale: fewer sizes and trials. + pub small: bool, + pub seed: u64, +} + +/// What one calibration measured. +#[derive(Debug, Clone)] +pub struct Calibration { + pub machine: MachineDescription, + pub costs: MeasuredOperationCosts, + pub accuracy: MeasuredSketchAccuracy, + /// `calibrate-run.json`: every case's samples and fit. + pub cost_fits: Vec, + /// `calibrate-run.json`: the errors per merge count. + pub accuracy_by_merge_count: Vec, +} + +/// Run a calibration on this machine. +pub fn calibrate(request: &CalibrateRequest) -> Result { + let _ = request; + todo!("asap_run: implement after test approval") +} + +/// Write `calibration` to `//`, replacing an earlier +/// run's files; returns that directory. +pub fn write_calibration(dir: &Path, calibration: &Calibration) -> Result { + let _ = (dir, calibration); + todo!("asap_run: implement after test approval") +} + +/// The tables a run was told to read on its command line. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TableOverrides { + /// `--costs `. + pub costs: Option, + /// `--accuracy `. + pub accuracy: Option, + /// `--cost-inputs `: cost inputs, possibly with a cost table + /// inside. + pub cost_inputs: Option, +} + +/// Where one table is read from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TablePath { + /// Named on the command line. + Given(PathBuf), + /// The machine's file in the calibration directory. + Calibrated(PathBuf), + /// No file: the run falls back to the formulas, warning that this + /// path was looked at. + Missing { looked_at: PathBuf }, +} + +/// Where a run's two tables are read from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedTablePaths { + pub costs: TablePath, + pub accuracy: TablePath, +} + +/// Where the tables of machine `machine_key` are read from: +/// `overrides` first, then the calibration directory (`env_dir`, from +/// [`CALIBRATION_DIR_ENV`], else `/calibration`). +pub fn resolve_table_paths( + overrides: &TableOverrides, + env_dir: Option<&Path>, + repo_root: Option<&Path>, + machine_key: &str, +) -> ResolvedTablePaths { + let _ = (overrides, env_dir, repo_root, machine_key); + todo!("asap_run: implement after test approval") +} + +/// The tables a planning run plans with. +#[derive(Debug, Clone)] +pub struct PlanningTables { + pub cost_inputs: CostInputs, + pub accuracy: MeasuredSketchAccuracy, + /// What the run prints to standard error, e.g. that no table exists + /// for this machine. + pub warnings: Vec, +} + +/// Why a table could not be read: a corrupt or invalid file. +#[derive(Debug, Clone, PartialEq)] +pub struct TableLoadError { + pub path: PathBuf, + pub reason: String, +} + +/// Load the tables `resolved` names, over the cost inputs +/// `overrides.cost_inputs` names (the defaults without one); a missing +/// table falls back to the formulas with a warning naming `machine_key` +/// and `asap_run calibrate`. +pub fn load_planning_tables( + resolved: &ResolvedTablePaths, + overrides: &TableOverrides, + machine_key: &str, +) -> Result { + let _ = (resolved, overrides, machine_key); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/cost_calibration.rs b/crates/devtools/src/measurement/cost_calibration.rs new file mode 100644 index 000000000..705507143 --- /dev/null +++ b/crates/devtools/src/measurement/cost_calibration.rs @@ -0,0 +1,165 @@ +//! Cost calibration (`asap_run calibrate`): microbenchmark every operator +//! kind × phase × state family on the executor, at several sizes, and fit +//! the planner's per-node model `cpu_ns = units · ns_per_unit + +//! ns_per_invocation` to the measurements. +//! +//! Example: an operator measured at 1,000, 10,000 and 100,000 units taking +//! 60,000, 510,000 and 5,010,000 ns fits `ns_per_unit = 50` and +//! `ns_per_invocation = 10,000`. + +use asap_cost::{MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_planner::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::summary_type::{ExactKind, SketchParams}; +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; + +/// One measurement: the units one invocation processed and the thread CPU +/// time it took. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct CostSample { + pub units: f64, + pub cpu_ns: f64, +} + +/// The fitted per-node model and what the fit noticed. +#[derive(Debug, Clone, PartialEq)] +pub struct FittedOperatorCost { + pub ns_per_unit: f64, + pub ns_per_invocation: f64, + /// Empty for a clean fit. + pub flags: Vec, +} + +/// Something about a fit `calibrate-run.json` records. +#[derive(Debug, Clone, PartialEq)] +pub enum FitFlag { + /// The unconstrained intercept was negative (`raw_intercept_ns`); it + /// was set to 0 and the slope refitted through the origin. + NegativeInterceptClamped { raw_intercept_ns: f64 }, + /// The fit misses the measurement at `units` by `relative_residual`, + /// more than the tolerance. + ResidualAboveTolerance { units: f64, relative_residual: f64 }, + /// Fewer distinct sizes, or a narrower span of sizes, than a fit needs. + TooFewSizes { sizes: usize }, +} + +/// Fit `cpu_ns = units · ns_per_unit + ns_per_invocation` to `samples`. +pub fn fit_operator_cost(samples: &[CostSample]) -> FittedOperatorCost { + let _ = samples; + todo!("asap_run: implement after test approval") +} + +/// A per-thread CPU clock. +pub trait CpuClock { + /// CPU nanoseconds this thread has used. + fn thread_cpu_ns(&self) -> u64; +} + +/// The operating system's per-thread CPU clock. +#[derive(Debug, Clone, Copy, Default)] +pub struct ThreadCpuClock; + +impl CpuClock for ThreadCpuClock { + fn thread_cpu_ns(&self) -> u64 { + todo!("asap_run: implement after test approval") + } +} + +/// One operator invocation a calibration times, at a given size. +pub trait CalibrationKernel { + /// Run one invocation processing `units` units, counted as the cost + /// model counts them (`OperatorKind`). + fn run(&mut self, units: u64); +} + +/// Time `kernel` on `clock` at each of `sizes`, `repetitions` times each, +/// one [`CostSample`] per size (the median of its repetitions). +pub fn measure_kernel( + kernel: &mut dyn CalibrationKernel, + clock: &dyn CpuClock, + sizes: &[u64], + repetitions: u32, +) -> Vec { + let _ = (kernel, clock, sizes, repetitions); + todo!("asap_run: implement after test approval") +} + +/// The state family an entry is for. +#[derive(Debug, Clone, PartialEq)] +pub enum CalibratedFamily { + None, + Sketch(SketchParams), + Exact(ExactKind), + /// An `Aggregate` measure, by its JSON `kind`. + Measure(String), +} + +/// One (operator, phase, family) a calibration measures. +#[derive(Debug, Clone, PartialEq)] +pub struct CalibrationCase { + pub operator: OperatorKind, + pub phase: ExecutionTiming, + pub family: CalibratedFamily, +} + +/// Which families a calibration measures. +#[derive(Debug, Clone, PartialEq)] +pub enum CalibrationFamilies { + All, + /// The exact accumulators and aggregate measures only. + Exact, + Only(Vec), +} + +/// The physical DAG a case's microbenchmark runs at one size. +#[derive(Debug, Clone)] +pub struct CalibrationDag { + pub dag: PhysicalASAPDAG, + pub entries: Vec, + pub data: DataWorkload, + /// The node the case times. + pub measured_node: PhysicalASAPNodeId, + /// The units that node processes per invocation, as the + /// microbenchmark counts them. + pub units: u64, +} + +/// Every case of `families`: each `OperatorKind` × phase × family. +pub fn calibration_cases(families: &CalibrationFamilies) -> Vec { + let _ = families; + todo!("asap_run: implement after test approval") +} + +impl CalibrationCase { + /// The DAG this case's microbenchmark runs at about `units` units. + pub fn dag_at(&self, units: u64) -> CalibrationDag { + let _ = units; + todo!("asap_run: implement after test approval") + } + + /// The table entry of this case with `fit`. + pub fn operator_cost( + &self, + fit: &FittedOperatorCost, + memory_bytes_per_state: Option, + ) -> OperatorCost { + let _ = (fit, memory_bytes_per_state); + todo!("asap_run: implement after test approval") + } +} + +/// One case's measurements and fit, as `calibrate-run.json` records them. +#[derive(Debug, Clone, PartialEq)] +pub struct CostFitRecord { + pub case: CalibrationCase, + pub samples: Vec, + pub fit: FittedOperatorCost, + pub memory_bytes_per_state: Option, +} + +/// The cost table of `records`, with the per-request overhead +/// `ns_per_evaluation`. +pub fn cost_table_of(records: &[CostFitRecord], ns_per_evaluation: f64) -> MeasuredOperationCosts { + let _ = (records, ns_per_evaluation); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/execution.rs b/crates/devtools/src/measurement/execution.rs new file mode 100644 index 000000000..8017111d5 --- /dev/null +++ b/crates/devtools/src/measurement/execution.rs @@ -0,0 +1,140 @@ +//! The reference driver of the executor (`asap_run execute`): plan a +//! workload, install the selected plan, ingest every refresh window in time +//! order (the executor's `execute_ingestion`, one ingestion-scope run per +//! window) and evaluate every query root at the requested times (its +//! `execute`, one query-scope run per evaluation time, whose root streams +//! are drained in the request's `RootDrainOrder`). +//! +//! Example: `sum by (job) (sum_over_time(m[1m]))` over four 1 Hz series of +//! constant values 1, 2 (`job="a"`) and 3, 4 (`job="b"`), evaluated at +//! t = 600 s, answers `{job="a"}: 180` and `{job="b"}: 420`: the window +//! `(540 s, 600 s]` holds 60 samples of each series. + +use std::collections::BTreeMap; + +use asap_accuracy::MeasuredSketchAccuracy; +use asap_cost::CostInputs; +use asap_planner::{ + DeploymentCapabilities, InstallablePlan, MaterializedViewRefresh, PhysicalASAPNodeId, + PlanningWorkload, SelectedPlanReport, +}; + +use crate::measurement::samples::{GeneratedDataSettings, Sample, SeriesLabels}; + +/// Where a run's samples come from. +#[derive(Debug, Clone, PartialEq)] +pub enum DataInput { + /// Generated by `aqpbm-datagen` from each source's `SourceFacts`. + GeneratedFromFacts(GeneratedDataSettings), + /// Given, e.g. read from a JSONL file. + Samples(Vec), +} + +/// In which order a query evaluation drains its roots' streams. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RootDrainOrder { + /// Root 0 first, then root 1, and so on. + WorkloadOrder, + /// The last root first. + Reversed, + /// Every root at once. + Concurrent, +} + +/// One `asap_run execute` run. +#[derive(Debug, Clone)] +pub struct ExecuteRequest<'a> { + pub workload: &'a PlanningWorkload, + pub data: DataInput, + /// Each evaluation time, in ms, ascending; ingestion starts at the + /// data's first sample. + pub evaluation_times_ms: Vec, + pub deployment_capabilities: DeploymentCapabilities, + pub cost_inputs: CostInputs, + pub accuracy: MeasuredSketchAccuracy, + pub root_drain_order: RootDrainOrder, +} + +impl<'a> ExecuteRequest<'a> { + /// A run of `workload` over `data` at `evaluation_times_ms`, with + /// raw data retained, the default cost inputs, no measured accuracy + /// and roots drained in workload order. + pub fn new( + workload: &'a PlanningWorkload, + data: DataInput, + evaluation_times_ms: Vec, + ) -> Self { + Self { + workload, + data, + evaluation_times_ms, + deployment_capabilities: DeploymentCapabilities::default(), + cost_inputs: CostInputs::default(), + accuracy: MeasuredSketchAccuracy::default(), + root_drain_order: RootDrainOrder::WorkloadOrder, + } + } +} + +/// What one run did and answered. +#[derive(Debug, Clone)] +pub struct ExecutionReport { + /// The plan installed. + pub plan: InstallablePlan, + /// Its predicted cost. + pub predicted: SelectedPlanReport, + /// One per evaluation time, in request order. + pub evaluations: Vec, + /// One per materialized view of the plan. + pub views: Vec, +} + +/// The answers of one evaluation. +#[derive(Debug, Clone, PartialEq)] +pub struct EvaluationAnswers { + pub at_ms: i64, + /// One per query root, in workload order. + pub per_root: Vec, +} + +/// One root's answer: a value per output series. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct RootAnswers { + pub values: BTreeMap, +} + +/// What one materialized view did over the run. +#[derive(Debug, Clone, PartialEq)] +pub struct ViewActivity { + pub node_id: PhysicalASAPNodeId, + pub refresh: MaterializedViewRefresh, + /// How many times it was refreshed. + pub refresh_count: u64, + /// The states it held after each evaluation, in evaluation order. + pub states_after_evaluation: Vec, + /// The bytes its states held after each evaluation (each kernel's + /// `memory_bytes()`). + pub bytes_after_evaluation: Vec, +} + +/// Why a run did not answer. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum ExecuteError { + /// Planning or installation failed. + Plan(String), + /// The data: unreadable, or not generable from the facts. + Data(String), + /// An evaluation at `at_ms` before the data covers its warm-up (the + /// longest retention or raw lookback plus the refresh width), which + /// starts at `first_covered_ms`. + BeforeWarmUp { at_ms: i64, first_covered_ms: i64 }, + /// The executor failed. + Executor(String), +} + +/// Run `request`: plan, install, ingest and evaluate. +pub fn execute(request: &ExecuteRequest<'_>) -> Result { + let _ = request; + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/machine.rs b/crates/devtools/src/measurement/machine.rs new file mode 100644 index 000000000..b5bae874b --- /dev/null +++ b/crates/devtools/src/measurement/machine.rs @@ -0,0 +1,73 @@ +//! The machine a calibration is measured on, and the key its tables are +//! stored under: CPU architecture, CPU model, logical cores and memory +//! type. Never the hostname and never the memory size. +//! +//! Example: an "Intel(R) Xeon(R) CPU E5-2683 v3 @ 2.00GHz" on x86_64 with 56 +//! logical cores and DDR4 registered DIMMs is +//! `x86_64-intel-xeon-e5-2683-v3-56c-registered-ddr4`. + +use serde::{Deserialize, Serialize}; + +/// The memory type of a machine whose DIMM types cannot be read (no EDAC, +/// as in a VM) and that has no override. +pub const UNKNOWN_MEMORY_TYPE: &str = "unknown-memory"; + +/// What `machine.json` records. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MachineDescription { + /// `std::env::consts::ARCH`, for example `x86_64`. + pub cpu_architecture: String, + pub cpu_model: String, + pub logical_cores: u32, + /// The DIMM type, for example `Registered-DDR4`; DIMM types that differ + /// are the sorted unique values joined by `-`; `UNKNOWN_MEMORY_TYPE` + /// when unreadable and not overridden. + pub memory_type: String, + /// `MemTotal`, in KiB. Recorded for information; not part of the key. + pub memory_kib: u64, + pub os: String, +} + +impl MachineDescription { + /// The machine `/proc/cpuinfo`, `/proc/meminfo` and the EDAC DIMM types + /// (`/sys/devices/system/edac/mc/mc*/dimm*/dimm_mem_type`, one string per + /// DIMM, empty without EDAC) describe. `memory_type_override` + /// (`--memory-type` / `ASAP_MEMORY_TYPE`) replaces the EDAC types. + pub fn from_proc( + cpu_architecture: &str, + cpuinfo: &str, + meminfo: &str, + dimm_memory_types: &[&str], + memory_type_override: Option<&str>, + os: &str, + ) -> Result { + let _ = ( + cpu_architecture, + cpuinfo, + meminfo, + dimm_memory_types, + memory_type_override, + os, + ); + todo!("asap_run: implement after test approval") + } + + /// This machine. + pub fn detect(memory_type_override: Option<&str>) -> Result { + let _ = memory_type_override; + todo!("asap_run: implement after test approval") + } + + /// The directory name its tables are stored under: `[a-z0-9-]+`, from + /// architecture, model, cores and memory type. + pub fn machine_key(&self) -> String { + todo!("asap_run: implement after test approval") + } + + /// A warning when the memory type is `UNKNOWN_MEMORY_TYPE`, naming it + /// and the `--memory-type` / `ASAP_MEMORY_TYPE` override. + pub fn warning(&self) -> Option { + todo!("asap_run: implement after test approval") + } +} diff --git a/crates/devtools/src/measurement/mod.rs b/crates/devtools/src/measurement/mod.rs new file mode 100644 index 000000000..98eb1c6fe --- /dev/null +++ b/crates/devtools/src/measurement/mod.rs @@ -0,0 +1,33 @@ +//! Running selected plans on the Planner's executor and measuring them, +//! behind the `asap_run` binary: +//! +//! - [`execution`]: the reference driver. Plan a workload, install the plan +//! (its materialized views and ingestion refresh groups), ingest each +//! refresh window, evaluate at given times and read every root's answers +//! (`asap_run execute`). +//! - [`samples`]: the data a run ingests, read from a JSONL file or +//! generated from the workload's source facts with `aqpbm-datagen`. +//! - [`cost_calibration`] and [`accuracy_calibration`]: microbenchmarks of +//! every operator, phase and state family, fitted into the planner's +//! `MeasuredOperationCosts`, and seeded trials of every sketch +//! configuration, recorded as `MeasuredSketchAccuracy` (`asap_run +//! calibrate`). +//! - [`machine`] and [`calibration_tables`]: the machine key a calibration +//! is stored under, and which tables a planning run reads. +//! - [`benchmark`]: each arm (exact, planned, no raw data) end to end, the +//! planner's predicted cost beside the measured one, and the gate +//! (`asap_run bench`). +//! - [`source_facts`]: `SourceFacts` computed from data, like a database's +//! `ANALYZE` (`asap_run facts`). +//! +//! The planner library only reads `CostInputs`, an `AccuracyEvidence` and +//! `SourceFacts`; every path, machine and data adapter lives here. + +pub mod accuracy_calibration; +pub mod benchmark; +pub mod calibration_tables; +pub mod cost_calibration; +pub mod execution; +pub mod machine; +pub mod samples; +pub mod source_facts; diff --git a/crates/devtools/src/measurement/samples.rs b/crates/devtools/src/measurement/samples.rs new file mode 100644 index 000000000..2e8da766e --- /dev/null +++ b/crates/devtools/src/measurement/samples.rs @@ -0,0 +1,79 @@ +//! The samples a run ingests: read from a JSONL file, one sample per line, +//! or generated with `aqpbm-datagen` from the workload's source facts. +//! +//! Example line: `{"labels": {"__name__": "m", "job": "a"}, "ts_ms": 1000, +//! "value": 2.0}`: series `m{job="a"}` has value 2 at t = 1 s. + +use std::collections::BTreeMap; + +use aqpbm_datagen::{GeneratedTimeSeries, TimeSeriesDescription}; +use asap_types::workload::{DataWorkload, SourceFacts}; +use serde::{Deserialize, Serialize}; + +/// A series' labels, the metric name under `__name__`. +pub type SeriesLabels = BTreeMap; + +/// The label holding a PromQL series' metric name. +pub const METRIC_NAME_LABEL: &str = "__name__"; + +/// One sample of one series. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Sample { + pub labels: SeriesLabels, + pub ts_ms: i64, + pub value: f64, +} + +/// Why a JSONL file is not a list of samples. +#[derive(Debug, Clone, PartialEq)] +pub struct SampleReadError { + /// 1-based line number. + pub line: usize, + pub reason: String, +} + +/// The samples of a JSONL text, in file order. +pub fn read_samples_jsonl(text: &str) -> Result, SampleReadError> { + let _ = text; + todo!("asap_run: implement after test approval") +} + +/// How `aqpbm-datagen` generates data from source facts. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct GeneratedDataSettings { + pub seed: u64, + pub start_ms: i64, + pub duration_ms: u64, +} + +/// The description `aqpbm-datagen` generates source `metric` from: +/// `series_count` series, one sample per `sample_interval`, each label's +/// `distinct_values_per_label` as its cardinality. +pub fn time_series_description_of( + metric: &str, + facts: &SourceFacts, + settings: &GeneratedDataSettings, +) -> Result { + let _ = (metric, facts, settings); + todo!("asap_run: implement after test approval") +} + +/// The samples of every source `data` declares, generated in time order. +pub fn generate_samples( + data: &DataWorkload, + settings: &GeneratedDataSettings, +) -> Result, String> { + let _ = (data, settings); + todo!("asap_run: implement after test approval") +} + +/// The samples of `generated`, as series of metric `metric`. An error when +/// the generator yields one, e.g. a counter whose running sum passed 2^53. +pub fn samples_of_generated( + metric: &str, + generated: &GeneratedTimeSeries, +) -> Result, String> { + let _ = (metric, generated); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/source_facts.rs b/crates/devtools/src/measurement/source_facts.rs new file mode 100644 index 000000000..090f05213 --- /dev/null +++ b/crates/devtools/src/measurement/source_facts.rs @@ -0,0 +1,108 @@ +//! Source facts from data (`asap_run facts`), like a database's `ANALYZE`: +//! per-source statistics over the samples, written into the workload's +//! `data_workload.sources`. +//! +//! Facts that affect only cost are written as observed. Facts a guarantee +//! relies on are printed as suggestions with a ×2 margin, and written only +//! when the caller trusts observed bounds: the next hour's data may exceed +//! an observed value, and then the guarantee is false. +//! +//! Example: four series `m{job, instance}` with samples 1 s apart give +//! `series_count: 4` and `sample_interval: 1000` (cost-only, written), and, +//! when a query has a whole-result target, a suggested +//! `distinct_values_per_label: {"job": 4}` for 2 observed job values. + +use std::collections::BTreeMap; + +use asap_types::stages::ValueSign; +use asap_types::workload::{DataWorkload, LabelSetDistinctValues, PlanningWorkload, StreamNorms}; + +use crate::measurement::samples::Sample; + +/// The smallest and largest non-zero absolute value of a column. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ObservedMagnitude { + pub min_abs: f64, + pub max_abs: f64, +} + +/// What one source's samples show. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ObservedSourceStatistics { + /// Distinct label sets. + pub series_count: u64, + /// The median of the per-series median gaps; `None` when no series has + /// two samples. + pub sample_interval_ms: Option, + /// Distinct values per label, a missing label counting as one more. + pub distinct_values_per_label: BTreeMap, + /// Distinct combinations of each `by (…)` set of two or more labels the + /// workload's queries use. + pub distinct_values_per_label_set: Vec, + /// Per column; a column of zeros only has none. + pub column_magnitudes: BTreeMap, + /// Per column, by sign bit; `Unknown` for mixed signs. + pub column_signs: BTreeMap, + /// The most distinct values one evaluation window of the workload's + /// longest window saw, per output group. + pub max_distinct_per_evaluation: Option, + /// The largest norms of one group's stream over the longest window. + pub stream_norms: Option, +} + +/// Each source's statistics over `samples`, keyed by metric name. +pub fn observe_source_statistics( + samples: &[Sample], + workload: &PlanningWorkload, +) -> BTreeMap { + let _ = (samples, workload); + todo!("asap_run: implement after test approval") +} + +/// One guarantee fact, suggested with its margin. +#[derive(Debug, Clone, PartialEq)] +pub enum SuggestedFact { + ColumnSign { column: String, sign: ValueSign }, + SeriesCount(u64), + DistinctValuesPerLabel { label: String, distinct_values: u64 }, + DistinctValuesPerLabelSet(LabelSetDistinctValues), + MaxDistinctPerEvaluation(u64), + StreamNorms(StreamNorms), +} + +/// A suggestion for one source. +#[derive(Debug, Clone, PartialEq)] +pub struct GuaranteeFactSuggestion { + pub source: String, + pub fact: SuggestedFact, +} + +/// How `asap_run facts` writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct FactsOptions { + /// `--trust-observed-bounds`: write the suggestions too. + pub trust_observed_bounds: bool, +} + +/// What `asap_run facts` produced. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceFactsReport { + pub observed: BTreeMap, + /// The workload's data workload with the facts written. + pub data_workload: DataWorkload, + /// The magnitudes written per source and column, with their margin; + /// they go into `SourceFacts::column_magnitudes` once it exists. + pub column_magnitudes: BTreeMap>, + /// The guarantee facts, printed; written too when trusted. + pub suggestions: Vec, +} + +/// The facts of `samples` for `workload`. +pub fn source_facts_of_samples( + samples: &[Sample], + workload: &PlanningWorkload, + options: FactsOptions, +) -> Result { + let _ = (samples, workload, options); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index ed12546d0..13ca11416 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -5,5 +5,6 @@ mod asap_planner; mod asap_planner_acceptance; mod cross_language; +mod measurement; mod planner_fuzz; mod viewer_contract; diff --git a/crates/devtools/tests/measurement/common.rs b/crates/devtools/tests/measurement/common.rs new file mode 100644 index 000000000..bd9c72cf0 --- /dev/null +++ b/crates/devtools/tests/measurement/common.rs @@ -0,0 +1,200 @@ +//! Fixtures shared by the `asap_run` tests: workloads, hand-made samples +//! and planning through the facade. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use asap_accuracy::AccuracyEvidence; +use asap_cost::CostInputs; +use asap_devtools::measurement::samples::{Sample, METRIC_NAME_LABEL}; +use asap_planner::{ + DeploymentCapabilities, InstallablePlan, PlanningModels, PlanningWorkload, SelectedPlanReport, +}; +use serde_json::{json, Value}; + +/// An exact accuracy requirement. +pub fn exact() -> Value { + json!({"max_error": 0.0, "failure_probability": 0.0}) +} + +/// Within `max_error` in `metric`, per value, failing with probability at +/// most `failure_probability`. +pub fn per_value(metric: &str, max_error: f64, failure_probability: f64) -> Value { + json!({ + "accuracy_metric": metric, + "max_error": max_error, + "failure_probability": failure_probability, + "accuracy_scope": "per_value", + }) +} + +/// Within `max_error` in `metric` for the whole result. +pub fn whole_result(metric: &str, max_error: f64, failure_probability: f64) -> Value { + json!({ + "accuracy_metric": metric, + "max_error": max_error, + "failure_probability": failure_probability, + "accuracy_scope": "whole_result", + }) +} + +/// The facts of a source of `series` series sampled every +/// `sample_interval_ms`. +pub fn facts(series: u64, sample_interval_ms: u64) -> Value { + json!({"series_count": series, "sample_interval": sample_interval_ms}) +} + +/// Repeating PromQL `queries` (each with its accuracy), every +/// `every_ms`, over `sources` (metric name → facts). The data arrives at +/// the rate the facts declare (each source's `series_count` per +/// `sample_interval`), unknown when none declares one. +pub fn workload( + queries: &[(&str, Value)], + every_ms: u64, + sources: &[(&str, Value)], +) -> PlanningWorkload { + let rows_per_second: f64 = sources + .iter() + .filter_map(|(_, facts)| { + Some(facts["series_count"].as_f64()? * 1_000.0 / facts["sample_interval"].as_f64()?) + }) + .sum(); + let arrival = match rows_per_second > 0.0 { + true => json!({"ingesting": {"rows_per_second": rows_per_second}}), + false => json!("unknown"), + }; + let value = json!({ + "query_workload": {"language": "promql", "query_batch": null, "repeating_queries": + queries.iter().map(|(query, accuracy)| json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": every_ms, "evaluation_phase": 0}}, + "requirements": {"accuracy": accuracy}, + "predictability": {"predictable": {"known_at": null}}, + })).collect::>()}, + "data_workload": { + "arrival": arrival, + "sources": sources.iter().map(|(name, facts)| (name.to_string(), facts.clone())) + .collect::>(), + }, + }); + serde_json::from_value(value).unwrap() +} + +/// One sample of metric `metric` with `labels`. +pub fn sample(metric: &str, labels: &[(&str, &str)], ts_ms: i64, value: f64) -> Sample { + let mut all: BTreeMap = labels + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + all.insert(METRIC_NAME_LABEL.into(), metric.into()); + Sample { + labels: all, + ts_ms, + value, + } +} + +/// The `job` label set of an answer. +pub fn job(name: &str) -> BTreeMap { + [("job".to_string(), name.to_string())].into() +} + +/// Four 1 Hz series of `m` from t = 0 to `until_ms` inclusive, of constant +/// values 1 and 2 (`job="a"`, instances 1 and 2) and 3 and 4 (`job="b"`), +/// with `value_of(series, ts_ms)` overriding a value (series 0 to 3). +pub fn four_constant_series( + until_ms: i64, + value_of: impl Fn(usize, i64) -> Option, +) -> Vec { + let series = [ + ("a", "1", 1.0), + ("a", "2", 2.0), + ("b", "3", 3.0), + ("b", "4", 4.0), + ]; + let mut samples = Vec::new(); + for ts_ms in (0..=until_ms).step_by(1_000) { + for (index, (job, instance, value)) in series.iter().enumerate() { + samples.push(sample( + "m", + &[("job", job), ("instance", instance)], + ts_ms, + value_of(index, ts_ms).unwrap_or(*value), + )); + } + } + samples +} + +/// The facts of [`four_constant_series`]. +pub fn four_series_facts() -> Value { + json!({ + "series_count": 4, + "sample_interval": 1000, + "distinct_values_per_label": {"job": 2, "instance": 4}, + }) +} + +/// Plan `workload` as a deployment runtime does. +pub fn plan( + workload: &PlanningWorkload, + cost_inputs: &CostInputs, + evidence: &dyn AccuracyEvidence, + capabilities: &DeploymentCapabilities, +) -> (InstallablePlan, SelectedPlanReport) { + let models = PlanningModels::new(evidence) + .with_cost_inputs(cost_inputs) + .with_deployment_capabilities(capabilities); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(asap_planner::plan_installable( + asap_planner::UserInput::new(workload, asap_planner::FrontendInput::Promql, models), + )) + .unwrap() +} + +/// Raw data not retained: every window is kept in ingestion-time state. +pub fn no_raw_data() -> DeploymentCapabilities { + DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + } +} + +/// A fresh empty directory under the system temp directory, removed when +/// dropped. +pub struct TempDir(pub PathBuf); + +impl TempDir { + pub fn new(name: &str) -> Self { + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "asap_run_test_{name}_{}_{call}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } + + pub fn write(&self, relative: &str, contents: &str) -> PathBuf { + let path = self.0.join(relative); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, contents).unwrap(); + path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Whether `a` and `b` agree to the exact-arm tolerance. +// provisional answer 21: exact plans agree within 1e-9 relative. +pub fn exact_match(a: f64, b: f64) -> bool { + (a - b).abs() <= 1e-9 * a.abs().max(b.abs()).max(1.0) +} diff --git a/crates/devtools/tests/measurement/cost_accounting.rs b/crates/devtools/tests/measurement/cost_accounting.rs new file mode 100644 index 000000000..d6f121da1 --- /dev/null +++ b/crates/devtools/tests/measurement/cost_accounting.rs @@ -0,0 +1,431 @@ +//! §2.3 and §2.8: the planner prices from the measured table, state +//! bytes, and a shared node charged once. + +use asap_accuracy::NoMeasuredGuarantees; +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::measurement::execution::{execute, DataInput, ExecuteRequest}; +use asap_executor::factory::create_planner_accumulator; +use asap_planner::{DeploymentCapabilities, MaterializedViewRefresh, PlanningWorkload}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchParams, SummaryInputExpr, SummaryUpdate, +}; + +use super::common::{ + exact, facts, four_constant_series, four_series_facts, no_raw_data, per_value, plan, workload, +}; + +/// DDSketch builds (`SummaryAgg` and the tumbling-window chain) at +/// `ns_per_unit` per row, both phases. +fn ddsketch_build_table(ns_per_unit: f64) -> MeasuredOperationCosts { + let entry = |operator| OperatorCost { + operator, + phase: None, + sketch: Some(SketchParams::DDSketch { alpha: 0.01 }), + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }; + MeasuredOperationCosts { + operators: vec![ + entry(OperatorKind::SummaryAgg), + entry(OperatorKind::SummaryTumblingWindowChain), + ], + ..MeasuredOperationCosts::default() + } +} + +/// `quantile_over_time(0.99, latency[1h])` every 10 s over 1,000 series +/// sampled each second, within 1% per value at δ = 0.01. +fn quantile_workload() -> PlanningWorkload { + workload( + &[( + "quantile_over_time(0.99, latency[1h])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("latency", facts(1_000, 1_000))], + ) +} + +/// The same plan priced with table A (DDSketch builds at 1 ns per row) and +/// with A's builds at 10 ns: each build node's CPU rises by its rows × 9 ns. +/// A query-time build reads 1,000 series × 3,600 samples per evaluation, +/// 0.1 evaluations per second: Δ = 3.6e6 · 0.1 · 9 ns = 3.24e-3 core-s/s. +/// An ingestion-time build reads the 1,000 arriving rows per second: +/// Δ = 1,000 · 9 ns = 9e-6 core-s/s. +#[test] +fn planner_prices_from_table() { + let workload = quantile_workload(); + let table_a = CostInputs { + measured_operation_costs: ddsketch_build_table(1.0), + ..CostInputs::default() + }; + let table_a10 = CostInputs { + measured_operation_costs: ddsketch_build_table(10.0), + ..CostInputs::default() + }; + let (installable, _) = plan( + &workload, + &table_a, + &NoMeasuredGuarantees, + &DeploymentCapabilities::default(), + ); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap(); + let (cost_a, _) = asap_cost::plan_cost(&installable.dag, &entries, &data, &table_a).unwrap(); + let (cost_a10, _) = + asap_cost::plan_cost(&installable.dag, &entries, &data, &table_a10).unwrap(); + let builds: Vec<_> = installable + .dag + .nodes + .iter() + .filter(|node| matches!(node.parameters, Some(SketchParams::DDSketch { .. }))) + .filter(|node| { + matches!( + node.payload, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } + ) + ) + }) + .collect(); + assert!( + !builds.is_empty(), + "the plan builds no DDSketch: {:?}", + installable.dag + ); + for node in builds { + let expected = match node.output_state.timing { + ExecutionTiming::QueryTime => 3.6e6 * 0.1 * 9e-9, + ExecutionTiming::IngestionTime => 1_000.0 * 9e-9, + }; + let delta = cost_a10.per_node[&node.id].cpu_core_seconds_per_second + - cost_a.per_node[&node.id].cpu_core_seconds_per_second; + assert!( + (delta - expected).abs() <= 1e-6 * expected, + "node {}: {delta} vs {expected}", + node.id + ); + } +} + +/// After a small calibration, every node of every small showcase plan is +/// priced from the table: no operator is left to the formulas. +#[test] +fn calibrated_table_covers_showcase_plans() { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: true, + seed: 7, + }) + .unwrap(); + let inputs = CostInputs { + measured_operation_costs: calibration.costs, + ..CostInputs::default() + }; + for (name, workload, settings) in asap_devtools::measurement::benchmark::showcase_suite(true) { + for setting in settings { + let caps = DeploymentCapabilities { + raw_data_retained: setting.raw_data_retained, + ..DeploymentCapabilities::default() + }; + let (_, report) = plan(&workload, &inputs, &calibration.accuracy, &caps); + for node in &report.predicted_usage.per_node { + assert_eq!( + node.cost_source, + asap_cost::NodeCostSource::Measured, + "{name} {setting:?}: {:?} at {:?}", + node.operator, + node.timing + ); + } + } + } +} + +/// `sum by (job) (sum_over_time(m[1h]))` every 10 s without raw data, +/// with a measured merge of 1 ns per state and 10 µs per invocation: a +/// 10 s chain merges 360 windows per evaluation, a 1 s chain 3,600, so the +/// planner keeps the 10 s chain. +/// +/// The plan's version of this test compares invocation costs at equal +/// units; the cost model charges an ingestion-time `ns_per_invocation` +/// per second of arriving rows, whatever the chain width, so here only +/// the merged states differ. +// provisional answer 10 +#[test] +fn invocation_cost_ranks_chain_widths() { + let merge = |phase| OperatorCost { + operator: OperatorKind::SummaryMerge, + phase: Some(phase), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 1.0, + ns_per_invocation: 10_000.0, + memory_bytes_per_state: None, + }; + let inputs = CostInputs { + measured_operation_costs: MeasuredOperationCosts { + operators: vec![ + merge(ExecutionTiming::QueryTime), + merge(ExecutionTiming::IngestionTime), + ], + ..MeasuredOperationCosts::default() + }, + ..CostInputs::default() + }; + let workload = workload( + &[("sum by (job) (sum_over_time(m[1h]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let (installable, _) = plan(&workload, &inputs, &NoMeasuredGuarantees, &no_raw_data()); + let widths: Vec = installable + .dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + _ => None, + }) + .collect(); + assert_eq!(widths, vec![10_000]); +} + +/// The bytes the executor charges one state of `family` with `parameters` +/// after one update: of the sample value, or (`keyed`) of one item. +fn kernel_state_bytes( + family: FieldDataType, + parameters: Option<&SketchParams>, + keyed: bool, +) -> u64 { + let value = SummaryUpdate::column(ColumnRef::SampleValue); + let input = match keyed { + false => value, + true => SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), + ..value + }, + }; + let mut updater = create_planner_accumulator(&family, parameters, &input).unwrap(); + match keyed { + false => updater.update_single(1.0, 0).unwrap(), + true => updater + .update_keyed(&["checkout".to_string()], 1.0, 0) + .unwrap(), + } + updater.into_accumulator().approx_memory_bytes() as u64 +} + +/// `sum by (job) (sum_over_time(m[1m]))` as a 10 s chain holds 6 windows × +/// 2 groups of one exact-sum state each: 12 · B_sum bytes, B_sum the +/// kernel's own charge for one state. The measured view bytes and the +/// planner's average memory are within 1.5× of it. +#[test] +fn memory_per_state_hand() { + let state_bytes = kernel_state_bytes( + FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + None, + false, + ); + let expected = 12.0 * state_bytes as f64; + let workload = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let mut request = ExecuteRequest::new( + &workload, + DataInput::Samples(four_constant_series(700_000, |_, _| None)), + vec![650_000, 700_000], + ); + request.deployment_capabilities = no_raw_data(); + let report = execute(&request).unwrap(); + let view = report + .views + .iter() + .find(|view| view.refresh == MaterializedViewRefresh::AtIngestion) + .expect("an ingestion-time view"); + let measured = *view.bytes_after_evaluation.last().unwrap() as f64; + assert!( + measured <= 1.5 * expected && expected <= 1.5 * measured, + "{measured} vs {expected}" + ); + let predicted = report.predicted.predicted_usage.average_memory_bytes; + assert!( + predicted <= 1.5 * expected && expected <= 1.5 * predicted, + "{predicted} vs {expected}" + ); +} + +/// Calibration writes, for a fixed-size state, the kernel's own charge: +/// Count-Min 272 × 5 holds what `create_planner_accumulator` charges +/// for it, at least d · w · 8 bytes of counters. +// provisional answer 9 +#[test] +fn fixed_size_state_bytes() { + let cms = SketchParams::Cms { + width: 272, + depth: 5, + }; + let charged = kernel_state_bytes( + FieldDataType::Sketch(SketchAlgorithm::Cms), + Some(&cms), + true, + ); + assert!(charged >= 272 * 5 * 8, "{charged}"); + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::Only(vec!["Cms".into()]), + small: true, + seed: 7, + }) + .unwrap(); + let entry = calibration + .costs + .operators + .iter() + .find(|entry| { + entry.operator == OperatorKind::SummaryAgg && entry.sketch.as_ref() == Some(&cms) + }) + .expect("an entry for Count-Min 272 x 5"); + assert_eq!(entry.memory_bytes_per_state, Some(charged as f64)); +} + +/// Two queries over one 10 s chain without raw data, +/// `sum by (job) (sum_over_time(m[5m]))` and the same over 10m: the joint +/// plan builds the chain once, and charges it once (one `per_node` entry, +/// read by both queries). +#[test] +fn shared_node_priced_once() { + let workload = workload( + &[ + ("sum by (job) (sum_over_time(m[5m]))", exact()), + ("sum by (job) (sum_over_time(m[10m]))", exact()), + ], + 10_000, + &[("m", four_series_facts())], + ); + let (installable, report) = plan( + &workload, + &CostInputs::default(), + &NoMeasuredGuarantees, + &no_raw_data(), + ); + let chains: Vec<_> = installable + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + }) + .collect(); + assert_eq!(chains.len(), 1, "{:?}", installable.dag); + let chain = chains[0].id; + let priced = report + .predicted_usage + .per_node + .iter() + .filter(|node| node.node_id == chain) + .count(); + assert_eq!(priced, 1); + // Both roots read the chain. + for &root in &installable.dag.roots { + assert!( + reaches(&installable.dag, root, chain), + "root {root} does not read the chain" + ); + } +} + +/// Whether `from` reaches `to` through its inputs. +fn reaches( + dag: &asap_planner::PhysicalASAPDAG, + from: asap_planner::PhysicalASAPNodeId, + to: asap_planner::PhysicalASAPNodeId, +) -> bool { + let mut stack = vec![from]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(node) = stack.pop() { + if node == to { + return true; + } + if seen.insert(node) { + stack.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == node) + .map(|edge| edge.producer), + ); + } + } + false +} + +/// The joint plan's measured ingestion CPU is about one chain's, not two +/// (≤ 1.2 × the single-query plan's). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn shared_node_measured_once() { + use asap_devtools::measurement::benchmark::{bench, BenchArm, BenchRequest, DataSetting}; + use asap_devtools::measurement::calibration_tables::PlanningTables; + let tables = PlanningTables { + cost_inputs: CostInputs::default(), + accuracy: Default::default(), + warnings: Vec::new(), + }; + let setting = DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms: 10_000, + raw_data_retained: false, + }; + let ingestion_cpu = |workload: &PlanningWorkload| { + let report = bench(&BenchRequest { + workload, + arms: vec![BenchArm::NoRaw], + settings: vec![setting.clone()], + series_count: 100, + duration_ms: 600_000, + repetitions: 5, + tables: tables.clone(), + seed: 1, + cost_gate: false, + output_root: None, + }) + .unwrap(); + report.arms[0] + .measured + .as_ref() + .unwrap() + .ingestion_cpu_seconds_per_second + }; + let single = workload( + &[("sum by (job) (sum_over_time(m[5m]))", exact())], + 10_000, + &[("m", facts(100, 1_000))], + ); + let joint = workload( + &[ + ("sum by (job) (sum_over_time(m[5m]))", exact()), + ("sum by (job) (sum_over_time(m[10m]))", exact()), + ], + 10_000, + &[("m", facts(100, 1_000))], + ); + let (single, joint) = (ingestion_cpu(&single), ingestion_cpu(&joint)); + assert!(joint <= 1.2 * single, "joint {joint} vs single {single}"); +} diff --git a/crates/devtools/tests/measurement/cost_fit.rs b/crates/devtools/tests/measurement/cost_fit.rs new file mode 100644 index 000000000..dc43b656e --- /dev/null +++ b/crates/devtools/tests/measurement/cost_fit.rs @@ -0,0 +1,239 @@ +//! §2.1: the cost fit `cpu_ns = units · ns_per_unit + ns_per_invocation` +//! over synthetic measurements of known coefficients, and the measurement +//! loop over a fake clock. + +use std::cell::Cell; +use std::rc::Rc; +use std::time::{Duration, Instant}; + +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::cost_calibration::{ + calibration_cases, fit_operator_cost, measure_kernel, CalibrationFamilies, CalibrationKernel, + CostSample, CpuClock, FitFlag, ThreadCpuClock, +}; + +fn samples(points: &[(f64, f64)]) -> Vec { + points + .iter() + .map(|&(units, cpu_ns)| CostSample { units, cpu_ns }) + .collect() +} + +/// The line through (1e3, 60,000), (1e4, 510,000), (1e5, 5,010,000) is +/// 50 ns per unit + 10,000 ns per invocation. +#[test] +fn fit_recovers_exact_line() { + let fit = fit_operator_cost(&samples(&[ + (1e3, 60_000.0), + (1e4, 510_000.0), + (1e5, 5_010_000.0), + ])); + assert!((fit.ns_per_unit - 50.0).abs() < 1e-9, "{fit:?}"); + assert!( + (fit.ns_per_invocation - 10_000.0).abs() < 1e-9 * 10_000.0, + "{fit:?}" + ); + assert!(fit.flags.is_empty(), "{fit:?}"); +} + +/// The same points × (1.02, 0.98, 1.01). Relative-weighted least squares +/// gives 49.69 / 11,391 (computed by hand from the normal equations with +/// weights 1/y²); plain least squares gives an intercept of 2,783, which +/// fails. +// provisional answer 11 +#[test] +fn fit_is_relative_weighted() { + let fit = fit_operator_cost(&samples(&[ + (1e3, 60_000.0 * 1.02), + (1e4, 510_000.0 * 0.98), + (1e5, 5_010_000.0 * 1.01), + ])); + assert!((48.5..=51.5).contains(&fit.ns_per_unit), "{fit:?}"); + assert!( + (8_000.0..=12_500.0).contains(&fit.ns_per_invocation), + "{fit:?}" + ); +} + +/// Five sizes from 10 to 1e5 units, noise +2%, −2%, +2%, −2%, +2%: the +/// weighted fit is 50.12 / 10,045 by hand, within 1% of 50 / 10,000. +// provisional answer 11 +#[test] +fn fit_five_sizes_tightens() { + let noise = [1.02, 0.98, 1.02, 0.98, 1.02]; + let points: Vec<(f64, f64)> = [10.0, 100.0, 1e3, 1e4, 1e5] + .iter() + .zip(noise) + .map(|(&units, noise)| (units, (50.0 * units + 10_000.0) * noise)) + .collect(); + let fit = fit_operator_cost(&samples(&points)); + assert!((fit.ns_per_unit - 50.0).abs() <= 0.5, "{fit:?}"); + assert!((fit.ns_per_invocation - 10_000.0).abs() <= 100.0, "{fit:?}"); +} + +/// (100, 900), (1000, 10,900) have intercept −211. The fit clamps it to +/// 0, refits through the origin (weighted: Σ(x/y) / Σ(x²/y²) = 9.7702), +/// flags the clamp, and the entry validates. +// provisional answer 11 +#[test] +fn fit_negative_intercept_is_clamped() { + let fit = fit_operator_cost(&samples(&[(100.0, 900.0), (1_000.0, 10_900.0)])); + assert_eq!(fit.ns_per_invocation, 0.0); + assert!( + (fit.ns_per_unit - 9.770_231_720_134_127).abs() < 1e-9, + "{fit:?}" + ); + assert!( + fit.flags.iter().any( + |flag| matches!(flag, FitFlag::NegativeInterceptClamped { raw_intercept_ns } + if (raw_intercept_ns + 211.111).abs() < 0.01) + ), + "{fit:?}" + ); + let inputs = CostInputs { + measured_operation_costs: MeasuredOperationCosts { + operators: vec![OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: fit.ns_per_unit, + ns_per_invocation: fit.ns_per_invocation, + memory_bytes_per_state: None, + }], + ..MeasuredOperationCosts::default() + }, + ..CostInputs::default() + }; + inputs.validate().unwrap(); +} + +/// A quadratic kernel (1e3, 1e4), (1e4, 1e5), (1e5, 1e7) misses a line by +/// far more than 10% at some point (−89% at 1e5 for the weighted line): +/// flagged, not silently accepted. +// provisional answer 11 +#[test] +fn fit_flags_nonlinear() { + let fit = fit_operator_cost(&samples(&[(1e3, 1e4), (1e4, 1e5), (1e5, 1e7)])); + assert!( + fit.flags + .iter() + .any(|flag| matches!(flag, FitFlag::ResidualAboveTolerance { .. })), + "{fit:?}" + ); +} + +/// A clock only the fake kernel advances. +struct FakeClock(Rc>); + +impl CpuClock for FakeClock { + fn thread_cpu_ns(&self) -> u64 { + self.0.get() + } +} + +/// Charges 50 ns per unit + 10 µs per invocation to the fake clock. +struct FakeKernel(Rc>); + +impl CalibrationKernel for FakeKernel { + fn run(&mut self, units: u64) { + self.0.set(self.0.get() + 50 * units + 10_000); + } +} + +/// Driven by the measurement loop at three sizes, a kernel of known cost +/// is recovered exactly: the loop times one invocation per sample, not a +/// batch, and reports the units the kernel processed. +#[test] +fn fit_recovers_fake_clock_kernel() { + let ticks = Rc::new(Cell::new(0)); + let clock = FakeClock(Rc::clone(&ticks)); + let mut kernel = FakeKernel(ticks); + let measured = measure_kernel(&mut kernel, &clock, &[1_000, 10_000, 100_000], 5); + assert_eq!( + measured, + samples(&[(1e3, 60_000.0), (1e4, 510_000.0), (1e5, 5_010_000.0)]) + ); + let fit = fit_operator_cost(&measured); + assert!((fit.ns_per_unit - 50.0).abs() < 1e-9, "{fit:?}"); + assert!((fit.ns_per_invocation - 10_000.0).abs() < 1e-6, "{fit:?}"); +} + +/// Busy-spins 50 ns per unit + 20 µs per invocation of wall time on this +/// thread, which is that much thread CPU time on an idle core. +struct SpinKernel; + +impl CalibrationKernel for SpinKernel { + fn run(&mut self, units: u64) { + let until = Instant::now() + Duration::from_nanos(50 * units + 20_000); + while Instant::now() < until { + std::hint::spin_loop(); + } + } +} + +/// On the thread CPU clock, a spinning kernel's coefficients come back +/// within 10% (per unit) and 25% (per invocation). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn fit_recovers_spin_kernel() { + let measured = measure_kernel( + &mut SpinKernel, + &ThreadCpuClock, + &[1_000, 10_000, 100_000], + 9, + ); + let fit = fit_operator_cost(&measured); + assert!((fit.ns_per_unit - 50.0).abs() <= 5.0, "{fit:?}"); + assert!( + (fit.ns_per_invocation - 20_000.0).abs() <= 5_000.0, + "{fit:?}" + ); +} + +/// Each case's microbenchmark counts the units the planner prices it by: +/// pricing the case's DAG with only the case's entry at 1 ns per unit and +/// none per invocation, the planner's measured node reports exactly the +/// microbenchmark's unit count (a scan's output rows, a merge's merged +/// states, any other node's first input rows). +#[test] +fn units_match_cost_model() { + for case in calibration_cases(&CalibrationFamilies::All) { + let calibration_dag = case.dag_at(1_000); + let fit = asap_devtools::measurement::cost_calibration::FittedOperatorCost { + ns_per_unit: 1.0, + ns_per_invocation: 0.0, + flags: Vec::new(), + }; + let inputs = CostInputs { + measured_operation_costs: MeasuredOperationCosts { + operators: vec![case.operator_cost(&fit, None)], + ..MeasuredOperationCosts::default() + }, + ..CostInputs::default() + }; + let (cost, _) = asap_cost::plan_cost( + &calibration_dag.dag, + &calibration_dag.entries, + &calibration_dag.data, + &inputs, + ) + .unwrap(); + let node = &cost.per_node[&calibration_dag.measured_node]; + assert_eq!( + node.cost_source, + asap_cost::NodeCostSource::Measured, + "{case:?}" + ); + // The planner writes "measured x ns + ns". + let units: u64 = node + .detail + .split("measured ") + .nth(1) + .and_then(|rest| rest.split(' ').next()) + .and_then(|units| units.parse().ok()) + .unwrap_or_else(|| panic!("{case:?}: {}", node.detail)); + assert_eq!(units, calibration_dag.units, "{case:?}"); + } +} diff --git a/crates/devtools/tests/measurement/end_to_end.rs b/crates/devtools/tests/measurement/end_to_end.rs new file mode 100644 index 000000000..905acdf24 --- /dev/null +++ b/crates/devtools/tests/measurement/end_to_end.rs @@ -0,0 +1,349 @@ +//! §2.12 and §2.13: the `asap_run` binary end to end on a tiny workload, +//! the showcase suite, and determinism under a seed. + +use std::process::Command; + +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, ZipfParameter}; +use asap_devtools::measurement::benchmark::{bench_showcase, GateVerdict}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest, PlanningTables}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::measurement::samples::{generate_samples, GeneratedDataSettings}; +use asap_devtools::measurement::source_facts::{source_facts_of_samples, FactsOptions}; + +use super::common::{exact, four_constant_series, four_series_facts, workload, TempDir}; + +fn asap_run(args: &[&str], dir: &TempDir) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_asap_run")) + .args(args) + .env("ASAP_CALIBRATION_DIR", &dir.0) + .current_dir(&dir.0) + .output() + .unwrap() +} + +fn jsonl(samples: &[asap_devtools::measurement::samples::Sample]) -> String { + samples + .iter() + .map(|sample| serde_json::to_string(sample).unwrap() + "\n") + .collect() +} + +/// `facts` on the hand-made JSONL, `calibrate --small --families exact`, +/// then `bench` on `sum by (job) (sum_over_time(m[1m]))`: the facts are the +/// data's, the costs validate, the bench answers 180 / 420, the gate +/// passes on answers (`--no-cost-gate`) and the run's files are written. +#[test] +fn e2e_tiny_hand_answers() { + let dir = TempDir::new("e2e"); + let data = dir.write( + "m.jsonl", + &jsonl(&four_constant_series(600_000, |_, _| None)), + ); + let plain = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[], + ); + let workload_path = dir.write("w.json", &serde_json::to_string(&plain).unwrap()); + let (data, workload_path) = (data.to_str().unwrap(), workload_path.to_str().unwrap()); + + let facts = asap_run( + &["facts", "--data", data, "--workload", workload_path], + &dir, + ); + assert!( + facts.status.success(), + "{}", + String::from_utf8_lossy(&facts.stderr) + ); + let with_facts: asap_planner::PlanningWorkload = + serde_json::from_slice(&facts.stdout).expect("facts writes the workload"); + let m = &with_facts.data_workload.as_ref().unwrap().sources["m"]; + assert_eq!(m.series_count, Some(4)); + assert_eq!(m.sample_interval.map(|interval| interval.0), Some(1_000)); + + let calibrated = asap_run(&["calibrate", "--small", "--families", "exact"], &dir); + assert!( + calibrated.status.success(), + "{}", + String::from_utf8_lossy(&calibrated.stderr) + ); + let costs = std::fs::read_dir(&dir.0) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path().join("costs.json")) + .find(|path| path.is_file()) + .expect("calibrate wrote /costs.json"); + let table: asap_cost::MeasuredOperationCosts = + serde_json::from_str(&std::fs::read_to_string(costs).unwrap()).unwrap(); + asap_cost::CostInputs { + measured_operation_costs: table, + ..asap_cost::CostInputs::default() + } + .validate() + .unwrap(); + + let facts_path = dir.write("w-facts.json", &serde_json::to_string(&with_facts).unwrap()); + let executed = asap_run( + &[ + "execute", + "--workload", + facts_path.to_str().unwrap(), + "--data", + data, + "--at", + "600000", + ], + &dir, + ); + assert!( + executed.status.success(), + "{}", + String::from_utf8_lossy(&executed.stderr) + ); + let answers = String::from_utf8_lossy(&executed.stdout); + assert!( + answers.contains("180") && answers.contains("420"), + "{answers}" + ); + + let benched = asap_run( + &[ + "bench", + "--workload", + facts_path.to_str().unwrap(), + "--arms", + "exact,planned,noraw", + "--duration", + "600000", + "--no-cost-gate", + ], + &dir, + ); + assert!( + benched.status.success(), + "{}", + String::from_utf8_lossy(&benched.stderr) + ); + let runs = dir.0.join("runs"); + assert!(runs.is_dir(), "bench wrote no runs/ directory"); +} + +/// `bench --suite showcase --small`: every row passes (from milestone 1). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn e2e_small_showcase() { + let report = bench_showcase(true, &calibrated(true), None).unwrap(); + assert_eq!(report.verdict, GateVerdict::Pass, "{:?}", report.rows); +} + +/// The full showcase meets the milestone criteria. +#[test] +#[ignore = "milestone scale"] +fn e2e_full_showcase() { + let report = bench_showcase(false, &calibrated(false), None).unwrap(); + assert_eq!(report.verdict, GateVerdict::Pass, "{:?}", report.rows); +} + +fn calibrated(small: bool) -> PlanningTables { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small, + seed: 7, + }) + .unwrap(); + PlanningTables { + cost_inputs: asap_cost::CostInputs { + measured_operation_costs: calibration.costs, + ..asap_cost::CostInputs::default() + }, + accuracy: calibration.accuracy, + warnings: Vec::new(), + } +} + +/// A hash of every sample `description` generates. +fn stream_hash(description: &TimeSeriesDescription) -> u64 { + use std::hash::{Hash, Hasher}; + let generated = description.generate().unwrap(); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for sample in generated.samples() { + let (labels, ts, value) = sample.unwrap(); + labels.hash(&mut hasher); + ts.hash(&mut hasher); + value.to_bits().hash(&mut hasher); + } + hasher.finish() +} + +/// `aqpbm-datagen`: one description and seed give one sample stream; a +/// different seed another. +#[test] +fn datagen_seeded_bit_identical() { + let description = TimeSeriesDescription { + labels: vec![ + SeriesLabelSpec { + name: "job".into(), + cardinality: 4, + skewness: 0.0, + }, + SeriesLabelSpec { + name: "instance".into(), + cardinality: 16, + skewness: 1.1, + }, + ], + series_num: 8, + start_ms: 0, + sample_interval_ms: 1_000, + duration_ms: 600_000, + value_distribution: DataDistribution::Zipf(ZipfParameter { + skewness: 1.1, + population_size: 1_000, + seed: 0, + }), + counter: false, + seed: 42, + }; + assert_eq!(stream_hash(&description), stream_hash(&description)); + let other = TimeSeriesDescription { + seed: 43, + ..description.clone() + }; + assert_ne!(stream_hash(&description), stream_hash(&other)); +} + +/// Data generated from the facts is identical across runs under one seed. +#[test] +fn generated_samples_seeded_bit_identical() { + let plain = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let data = plain.data_workload.clone().unwrap(); + let settings = GeneratedDataSettings { + seed: 42, + start_ms: 0, + duration_ms: 600_000, + }; + assert_eq!( + generate_samples(&data, &settings).unwrap(), + generate_samples(&data, &settings).unwrap() + ); +} + +/// `facts` twice gives byte-identical JSON. +#[test] +fn facts_seeded_bit_identical() { + let samples = four_constant_series(600_000, |series, ts| { + (series == 1).then_some((ts % 13_000) as f64) + }); + let plain = workload( + &[("sum by (job, instance) (sum_over_time(m[1m]))", exact())], + 10_000, + &[], + ); + let once = || { + let report = source_facts_of_samples(&samples, &plain, FactsOptions::default()).unwrap(); + serde_json::to_string(&report.data_workload).unwrap() + }; + assert_eq!(once(), once()); +} + +/// Two small calibrations write byte-identical accuracy tables (trials +/// are seeded). +#[test] +fn calibrate_accuracy_seeded() { + let once = || { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: true, + seed: 7, + }) + .unwrap(); + serde_json::to_string(&calibration.accuracy).unwrap() + }; + assert_eq!(once(), once()); +} + +/// The same tables give the same `plan.txt`. +#[test] +fn plan_given_tables_identical() { + let dir = TempDir::new("plan_txt"); + let plain = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let workload_path = dir.write("w.json", &serde_json::to_string(&plain).unwrap()); + for _ in 0..2 { + let output = asap_run( + &[ + "bench", + "--workload", + workload_path.to_str().unwrap(), + "--arms", + "planned", + "--duration", + "600000", + "--no-cost-gate", + ], + &dir, + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let runs = dir.0.join("runs"); + let machine = std::fs::read_dir(&runs) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let plans: Vec = std::fs::read_dir(machine) + .unwrap() + .map(|run| std::fs::read_to_string(run.unwrap().path().join("plan.txt")).unwrap()) + .collect(); + assert_eq!(plans.len(), 2, "two runs, two run directories"); + assert_eq!(plans[0], plans[1]); +} + +/// Two full calibrations: the costs differ by at most the noise and the +/// showcase plan selection is identical, or each change still passes the +/// gate. +// provisional answer 23 +#[test] +#[ignore = "milestone scale"] +fn plan_stable_under_recalibration() { + let first = calibrated(false); + let second = calibrated(false); + for (a, b) in first + .cost_inputs + .measured_operation_costs + .operators + .iter() + .zip(&second.cost_inputs.measured_operation_costs.operators) + { + assert_eq!( + (a.operator, &a.phase, &a.sketch, &a.exact, &a.measure), + (b.operator, &b.phase, &b.sketch, &b.exact, &b.measure) + ); + let ratio = a.ns_per_unit.max(1e-9) / b.ns_per_unit.max(1e-9); + assert!((1.0 / 1.05..=1.05).contains(&ratio), "{a:?} vs {b:?}"); + } + let first_run = bench_showcase(false, &first, None).unwrap(); + let second_run = bench_showcase(false, &second, None).unwrap(); + for (a, b) in first_run.arms.iter().zip(&second_run.arms) { + if a.plan != b.plan { + assert_eq!( + second_run.verdict, + GateVerdict::Pass, + "a changed plan fails the gate" + ); + } + } +} diff --git a/crates/devtools/tests/measurement/execution.rs b/crates/devtools/tests/measurement/execution.rs new file mode 100644 index 000000000..fd423ad27 --- /dev/null +++ b/crates/devtools/tests/measurement/execution.rs @@ -0,0 +1,373 @@ +//! §2.7: the reference driver over hand-made data. Four 1 Hz series of +//! `m`, constant 1 and 2 (`job="a"`) and 3 and 4 (`job="b"`); every +//! expected answer is counted by hand over the window `(t − range, t]`. + +use std::sync::mpsc; +use std::time::Duration; + +use asap_devtools::measurement::execution::{ + execute, DataInput, EvaluationAnswers, ExecuteError, ExecuteRequest, ExecutionReport, + RootDrainOrder, +}; +use asap_devtools::measurement::samples::Sample; +use asap_planner::{DeploymentCapabilities, MaterializedViewRefresh, PlanningWorkload}; + +use super::common::{ + exact, exact_match, facts, four_constant_series, four_series_facts, job, no_raw_data, + per_value, sample, workload, +}; + +const SUM_BY_JOB: &str = "sum by (job) (sum_over_time(m[1m]))"; + +fn sum_by_job() -> PlanningWorkload { + workload( + &[(SUM_BY_JOB, exact())], + 10_000, + &[("m", four_series_facts())], + ) +} + +fn run(request: &ExecuteRequest<'_>) -> ExecutionReport { + execute(request).unwrap() +} + +/// Root `root`'s value for `job` at evaluation `evaluation`. +fn value(report: &ExecutionReport, evaluation: usize, root: usize, name: &str) -> f64 { + report.evaluations[evaluation].per_root[root].values[&job(name)] +} + +/// At t = 600 s the window (540 s, 600 s] holds 60 samples per series: +/// a: 60 · (1 + 2) = 180, b: 60 · (3 + 4) = 420. A window closed on both +/// ends would hold 61 (183 / 427). +#[test] +fn execute_sum_over_time_hand() { + let workload = sum_by_job(); + let request = ExecuteRequest::new( + &workload, + DataInput::Samples(four_constant_series(600_000, |_, _| None)), + vec![600_000], + ); + let report = run(&request); + assert_eq!(value(&report, 0, 0, "a"), 180.0); + assert_eq!(value(&report, 0, 0, "b"), 420.0); + assert_eq!(report.evaluations[0].per_root[0].values.len(), 2); +} + +/// Series 1's sample at exactly t − 60 s = 540 s is 1000: excluded, so a +/// is still 180; the sample at t = 600 s is included (else 177). +#[test] +fn execute_window_boundary() { + let data = four_constant_series(600_000, |series, ts| { + (series == 0 && ts == 540_000).then_some(1_000.0) + }); + let workload = sum_by_job(); + let request = ExecuteRequest::new(&workload, DataInput::Samples(data), vec![600_000]); + let report = run(&request); + assert_eq!(value(&report, 0, 0, "a"), 180.0); +} + +/// A counter `c_total` = t / 1000 on every series rises 1 per second: +/// `sum by (job) (rate(c_total[1m]))` is 2.0 for each job. +#[test] +fn execute_rate_linear_counter() { + let mut samples = Vec::new(); + for ts in (0..=600_000).step_by(1_000) { + for (job, instance) in [("a", "1"), ("a", "2"), ("b", "3"), ("b", "4")] { + samples.push(sample( + "c_total", + &[("job", job), ("instance", instance)], + ts, + ts as f64 / 1000.0, + )); + } + } + let workload = workload( + &[("sum by (job) (rate(c_total[1m]))", exact())], + 10_000, + &[("c_total", four_series_facts())], + ); + let report = run(&ExecuteRequest::new( + &workload, + DataInput::Samples(samples), + vec![600_000], + )); + assert!(exact_match(value(&report, 0, 0, "a"), 2.0), "{report:?}"); + assert!(exact_match(value(&report, 0, 0, "b"), 2.0), "{report:?}"); +} + +/// Series 1 changes to 10 after t = 605 s. At 600 s: a = 180; at 610 s the +/// window holds 5 samples of 10: 180 − 5 + 50 = 225; at 620 s, 15: +/// 180 − 15 + 150 = 315. b stays 420. +#[test] +fn execute_several_eval_times() { + let data = four_constant_series(620_000, |series, ts| { + (series == 0 && ts > 605_000).then_some(10.0) + }); + let workload = sum_by_job(); + let request = ExecuteRequest::new( + &workload, + DataInput::Samples(data), + vec![600_000, 610_000, 620_000], + ); + let report = run(&request); + let a: Vec = (0..3).map(|i| value(&report, i, 0, "a")).collect(); + assert_eq!(a, vec![180.0, 225.0, 315.0]); + assert!((0..3).all(|i| value(&report, i, 0, "b") == 420.0)); +} + +/// The same query planned at query time over raw data and as an +/// ingestion-time chain (no raw data) answers the same. +// provisional answer 21 +#[test] +fn execute_ingestion_equals_query_time() { + let data = four_constant_series(620_000, |series, ts| { + (series == 0 && ts > 605_000).then_some(10.0) + }); + let workload = sum_by_job(); + let times = vec![600_000, 610_000, 620_000]; + let query_time = run(&ExecuteRequest::new( + &workload, + DataInput::Samples(data.clone()), + times.clone(), + )); + let mut chained = ExecuteRequest::new(&workload, DataInput::Samples(data), times); + chained.deployment_capabilities = no_raw_data(); + let chained = run(&chained); + for (left, right) in query_time.evaluations.iter().zip(&chained.evaluations) { + for (l, r) in left.per_root.iter().zip(&right.per_root) { + assert_eq!( + l.values.keys().collect::>(), + r.values.keys().collect::>() + ); + for (key, value) in &l.values { + assert!( + exact_match(*value, r.values[key]), + "{key:?}: {value} vs {}", + r.values[key] + ); + } + } + } +} + +/// Two queries over one producer: `sum` and `count` of each job's +/// samples in the last minute. +fn joint_workload() -> PlanningWorkload { + workload( + &[ + (SUM_BY_JOB, exact()), + ("sum by (job) (count_over_time(m[1m]))", exact()), + ], + 10_000, + &[("m", four_series_facts())], + ) +} + +/// Run `workload` over `data` at t = 600 s with `order` on another thread +/// and return its answers, failing after 10 s: a deadlock guard, not a +/// timing assertion. +fn answers_with_deadlock_guard( + workload: &'static PlanningWorkload, + data: Vec, + order: RootDrainOrder, +) -> Vec { + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + let mut request = ExecuteRequest::new(workload, DataInput::Samples(data), vec![600_000]); + request.root_drain_order = order; + let _ = sender.send(execute(&request).map(|report| report.evaluations)); + }); + receiver + .recv_timeout(Duration::from_secs(10)) + .expect("the run did not finish within 10 s") + .unwrap() +} + +/// Draining the roots of a joint plan in any order gives the same answers +/// and finishes: sum 180 / 420, count 120 / 120. +#[test] +fn execute_drain_order() { + let workload: &'static PlanningWorkload = Box::leak(Box::new(joint_workload())); + let data = four_constant_series(600_000, |_, _| None); + let mut answers = Vec::new(); + for order in [ + RootDrainOrder::Reversed, + RootDrainOrder::WorkloadOrder, + RootDrainOrder::Concurrent, + ] { + let evaluations = answers_with_deadlock_guard(workload, data.clone(), order); + assert_eq!(evaluations[0].per_root[0].values[&job("a")], 180.0); + assert_eq!(evaluations[0].per_root[1].values[&job("a")], 120.0); + assert_eq!(evaluations[0].per_root[1].values[&job("b")], 120.0); + answers.push(evaluations); + } + assert!(answers.windows(2).all(|pair| pair[0] == pair[1])); +} + +/// Without raw data both queries read ingestion-time views fed by one +/// scan: at ingestion time the scan runs once per window and each batch +/// reaches both chain builds, so both views are complete (sum 180 / 420, +/// count 120 / 120), not one view missing the batches the other read first. +#[test] +fn execute_shared_ingestion_scan_feeds_every_consumer() { + let mut request = ExecuteRequest::new( + Box::leak(Box::new(joint_workload())), + DataInput::Samples(four_constant_series(600_000, |_, _| None)), + vec![600_000], + ); + request.deployment_capabilities = no_raw_data(); + let report = run(&request); + assert_eq!(value(&report, 0, 0, "a"), 180.0); + assert_eq!(value(&report, 0, 0, "b"), 420.0); + assert_eq!(value(&report, 0, 1, "a"), 120.0); + assert_eq!(value(&report, 0, 1, "b"), 120.0); +} + +/// A 1m window over a 10 s chain evaluated 20 times every 10 s: after the +/// warm-up the view holds a constant number of states, 6 windows × 2 +/// groups. +#[test] +fn execute_retention_evicts() { + let times: Vec = (0..20).map(|i| 600_000 + i * 10_000).collect(); + let mut request = ExecuteRequest::new( + Box::leak(Box::new(sum_by_job())), + DataInput::Samples(four_constant_series(*times.last().unwrap(), |_, _| None)), + times, + ); + request.deployment_capabilities = no_raw_data(); + let report = run(&request); + let chain = report + .views + .iter() + .find(|view| view.refresh == MaterializedViewRefresh::AtIngestion) + .expect("an ingestion-time view"); + let after_warm_up = &chain.states_after_evaluation[1..]; + assert!( + after_warm_up.iter().all(|&states| states == 12), + "{after_warm_up:?}" + ); +} + +/// An evaluation at t = 30 s, before a 1m window of data exists, is +/// refused, never answered from half a window. +// provisional answer 18 +#[test] +fn execute_before_warmup() { + let workload = sum_by_job(); + let request = ExecuteRequest::new( + &workload, + DataInput::Samples(four_constant_series(600_000, |_, _| None)), + vec![30_000], + ); + assert!(matches!( + execute(&request), + Err(ExecuteError::BeforeWarmUp { at_ms: 30_000, .. }) + )); +} + +/// A plan with a view refreshed on first read (query-time results kept +/// across evaluations) is run, not refused: the view is refreshed once +/// and the second evaluation reuses it. +#[test] +fn execute_on_first_read_view() { + let workload = workload( + &[("sum by (job) (sum_over_time(m[1m] offset 1m))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let mut request = ExecuteRequest::new( + Box::leak(Box::new(workload)), + DataInput::Samples(four_constant_series(620_000, |_, _| None)), + vec![610_000, 620_000], + ); + request.deployment_capabilities = DeploymentCapabilities { + query_time_results_retained: true, + ..DeploymentCapabilities::default() + }; + let report = run(&request); + let view = report + .views + .iter() + .find(|view| view.refresh == MaterializedViewRefresh::OnFirstRead) + .expect("the plan has an on-first-read view"); + assert_eq!(view.refresh_count, 1, "{view:?}"); + assert_eq!(value(&report, 1, 0, "a"), 180.0); +} + +/// `quantile_over_time(0.5, m[1m])` over values 1 … 60: exactly, 30.5 (the +/// interpolated median); a DDSketch with α = 0.01 answers within 1% of the +/// nearest-rank bracket [30, 31]. +// provisional answer 4 +#[test] +fn execute_quantile_reference() { + // Values 1 … 60 in the window (540 s, 600 s]. + let samples: Vec = (0..=600) + .map(|second| { + let value = if second > 540 { + (second - 540) as f64 + } else { + 1.0 + }; + sample("lat", &[("job", "a")], second * 1_000, value) + }) + .collect(); + let query = "quantile_over_time(0.5, lat[1m])"; + let exact_workload = workload(&[(query, exact())], 10_000, &[("lat", facts(1, 1_000))]); + let exact_report = run(&ExecuteRequest::new( + &exact_workload, + DataInput::Samples(samples.clone()), + vec![600_000], + )); + assert_eq!(value(&exact_report, 0, 0, "a"), 30.5); + let sketch_workload = workload( + &[(query, per_value("relative", 0.01, 0.0))], + 10_000, + &[("lat", facts(1, 1_000))], + ); + let sketch_report = run(&ExecuteRequest::new( + &sketch_workload, + DataInput::Samples(samples), + vec![600_000], + )); + let answer = value(&sketch_report, 0, 0, "a"); + assert!((30.0 * 0.99..=31.0 * 1.01).contains(&answer), "{answer}"); +} + +/// Two runs, and two drain orders, give identical answer maps (I9). +#[test] +fn execute_answers_bit_identical() { + let workload: &'static PlanningWorkload = Box::leak(Box::new(joint_workload())); + let data = four_constant_series(620_000, |series, ts| { + (series == 2 && ts % 7_000 == 0).then_some(0.1 * ts as f64) + }); + let mut runs = Vec::new(); + for order in [ + RootDrainOrder::WorkloadOrder, + RootDrainOrder::WorkloadOrder, + RootDrainOrder::Reversed, + ] { + let mut request = ExecuteRequest::new( + workload, + DataInput::Samples(data.clone()), + vec![600_000, 620_000], + ); + request.root_drain_order = order; + runs.push(run(&request).evaluations); + } + for pair in runs.windows(2) { + for (left, right) in pair[0].iter().zip(&pair[1]) { + for (l, r) in left.per_root.iter().zip(&right.per_root) { + assert_eq!( + l.values + .iter() + .map(|(k, v)| (k.clone(), v.to_bits())) + .collect::>(), + r.values + .iter() + .map(|(k, v)| (k.clone(), v.to_bits())) + .collect::>() + ); + } + } + } +} diff --git a/crates/devtools/tests/measurement/gate.rs b/crates/devtools/tests/measurement/gate.rs new file mode 100644 index 000000000..862d11868 --- /dev/null +++ b/crates/devtools/tests/measurement/gate.rs @@ -0,0 +1,298 @@ +//! §2.9: the bench gate's rule, and runs whose tables lie. + +use std::path::PathBuf; + +use asap_accuracy::{ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy}; +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::benchmark::{ + bench, gate_verdict, BenchArm, BenchRequest, DataSetting, GateFailure, GateRow, GateVerdict, +}; +use asap_devtools::measurement::calibration_tables::PlanningTables; +use asap_planner::PlanningWorkload; +use asap_types::accuracy::AccuracyMetric; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +use super::common::{facts, per_value, workload, TempDir}; + +fn setting() -> DataSetting { + DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms: 10_000, + raw_data_retained: true, + } +} + +fn row(exact: Option, selected: Result, error: f64, target: f64) -> GateRow { + GateRow { + workload: "w".into(), + setting: setting(), + exact_cost_per_second: exact, + selected_cost_per_second: selected, + measured_error: error, + target_error: target, + } +} + +fn verdict_of(row: GateRow) -> GateVerdict { + gate_verdict(&[("w".into(), setting())], &[row], true) +} + +fn fails(verdict: &GateVerdict) -> bool { + matches!(verdict, GateVerdict::Fail(failures) if !failures.is_empty()) +} + +/// The threshold arithmetic: cost 105 against exact 100 passes (≤ 1.05 ×), +/// 105.01 fails; error 0.0100 against 0.01 passes (`within_limit`), 0.0101 +/// fails; a missing exact arm, a failed selected arm, a NaN error, and a +/// missing or extra row each fail. +// provisional answer 1 +#[test] +fn gate_rule_pure() { + assert_eq!( + verdict_of(row(Some(100.0), Ok(105.0), 0.0, 0.01)), + GateVerdict::Pass + ); + assert!(fails(&verdict_of(row(Some(100.0), Ok(105.01), 0.0, 0.01)))); + assert_eq!( + verdict_of(row(Some(100.0), Ok(50.0), 0.0100, 0.01)), + GateVerdict::Pass + ); + assert!(fails(&verdict_of(row(Some(100.0), Ok(50.0), 0.0101, 0.01)))); + assert!(matches!( + verdict_of(row(None, Ok(50.0), 0.0, 0.01)), + GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::ExactArmMissing { .. })) + )); + assert!(matches!( + verdict_of(row(Some(100.0), Err("panicked".into()), 0.0, 0.01)), + GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::SelectedArmFailed { .. })) + )); + assert!(matches!( + verdict_of(row(Some(100.0), Ok(50.0), f64::NAN, 0.01)), + GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::ErrorNotANumber { .. })) + )); + let missing = gate_verdict( + &[("w".into(), setting()), ("v".into(), setting())], + &[row(Some(100.0), Ok(50.0), 0.0, 0.01)], + true, + ); + assert!( + matches!(&missing, GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::MissingRow { .. }))) + ); + let extra = gate_verdict(&[], &[row(Some(100.0), Ok(50.0), 0.0, 0.01)], true); + assert!( + matches!(&extra, GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::ExtraRow { .. }))) + ); + // Without the cost gate, a costly plan with a met target passes. + assert_eq!( + gate_verdict( + &[("w".into(), setting())], + &[row(Some(100.0), Ok(500.0), 0.0, 0.01)], + false + ), + GateVerdict::Pass + ); +} + +fn tables(costs: MeasuredOperationCosts, accuracy: MeasuredSketchAccuracy) -> PlanningTables { + PlanningTables { + cost_inputs: CostInputs { + measured_operation_costs: costs, + ..CostInputs::default() + }, + accuracy, + warnings: Vec::new(), + } +} + +fn request<'a>( + workload: &'a PlanningWorkload, + tables: PlanningTables, + output_root: Option, +) -> BenchRequest<'a> { + BenchRequest { + workload, + arms: vec![BenchArm::Exact, BenchArm::Planned], + settings: vec![setting()], + series_count: 100, + duration_ms: 600_000, + repetitions: 5, + tables, + seed: 1, + cost_gate: true, + output_root, + } +} + +/// A cost table claiming the exact scan costs 1e4 ns per row and the +/// DDSketch build 1e-3 ns per row makes the planner pick the DDSketch +/// query-time build, which measures about 2× the exact plan (217 against +/// 82–107 ns per row in the feasibility run): the gate fails. +// provisional answer 1 and provisional answer 23 +#[test] +#[ignore = "timing: run on a dedicated host"] +fn gate_fails_on_lying_cost_table() { + let workload = workload( + &[( + "quantile_over_time(0.99, m[10m])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("m", facts(100, 1_000))], + ); + let lie = |operator, sketch, ns_per_unit| OperatorCost { + operator, + phase: None, + sketch, + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }; + let costs = MeasuredOperationCosts { + operators: vec![ + lie(OperatorKind::Scan, None, 1e4), + lie( + OperatorKind::SummaryAgg, + Some(SketchParams::DDSketch { alpha: 0.01 }), + 1e-3, + ), + ], + ..MeasuredOperationCosts::default() + }; + let report = bench(&request( + &workload, + tables(costs, MeasuredSketchAccuracy::default()), + None, + )) + .unwrap(); + assert!( + matches!(&report.verdict, GateVerdict::Fail(failures) + if failures.iter().any(|f| matches!(f, GateFailure::CostAboveLimit { .. }))), + "{:?}", + report.verdict + ); +} + +/// An accuracy table claiming KLL k = 26 has rank error 1e-4 makes the +/// planner pick k = 26 for rank target 0.005 at δ = 0.01; its measured +/// rank error is about 0.05–0.1: the gate fails on error (deterministic +/// under a seed). +// provisional answer 3 +#[test] +fn gate_fails_on_lying_accuracy_table() { + let workload = workload( + &[( + "quantile_over_time(0.99, m[10m])", + per_value("rank", 0.005, 0.01), + )], + 10_000, + &[("m", facts(100, 1_000))], + ); + let accuracy = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k: 26 }, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric: AccuracyMetric::Rank, + trials: 1_000, + error_at: vec![ErrorAtFailureProbability { + failure_probability: 0.01, + error: 1e-4, + }], + }], + }; + let mut request = request( + &workload, + tables(MeasuredOperationCosts::default(), accuracy), + None, + ); + request.cost_gate = false; + let report = bench(&request).unwrap(); + let planned = report + .arms + .iter() + .find(|arm| arm.arm == BenchArm::Planned) + .unwrap(); + assert!( + planned + .plan + .dag + .nodes + .iter() + .any(|node| node.parameters == Some(SketchParams::Kll { k: 26 })), + "the lying table did not select k = 26" + ); + assert!( + matches!(&report.verdict, GateVerdict::Fail(failures) + if failures.iter().any(|f| matches!(f, GateFailure::ErrorAboveTarget { .. }))), + "{:?}", + report.verdict + ); +} + +/// The 1m-window counterexample: `quantile_over_time(0.99, m[1m])` at 1 Hz +/// every 10 s selects exact, or a plan within 1.05× of it: the gate passes. +// provisional answer 23 +#[test] +#[ignore = "timing: run on a dedicated host"] +fn gate_counterexample_passes_by_exact() { + let workload = workload( + &[( + "quantile_over_time(0.99, m[1m])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("m", facts(100, 1_000))], + ); + let report = bench(&request( + &workload, + tables( + MeasuredOperationCosts::default(), + MeasuredSketchAccuracy::default(), + ), + None, + )) + .unwrap(); + assert_eq!(report.verdict, GateVerdict::Pass); +} + +/// A run writes `runs///{workload.json, plan.txt, +/// results.csv, summary.md}`, one results row per workload and setting. +#[test] +fn gate_output_files() { + let root = TempDir::new("runs"); + let workload = workload( + &[( + "sum by (job) (sum_over_time(m[1m]))", + super::common::exact(), + )], + 10_000, + &[("m", super::common::four_series_facts())], + ); + let mut request = request( + &workload, + tables( + MeasuredOperationCosts::default(), + MeasuredSketchAccuracy::default(), + ), + Some(root.0.clone()), + ); + request.series_count = 4; + request.cost_gate = false; + let report = bench(&request).unwrap(); + let run_dir = report.run_dir.expect("the run was written"); + assert!(run_dir.starts_with(root.0.join("runs")), "{run_dir:?}"); + assert_eq!( + run_dir.parent().unwrap().parent().unwrap(), + root.0.join("runs") + ); + for file in ["workload.json", "plan.txt", "results.csv", "summary.md"] { + assert!( + run_dir.join(file).is_file(), + "{file} missing in {run_dir:?}" + ); + } + let results = std::fs::read_to_string(run_dir.join("results.csv")).unwrap(); + // A header and one row for one workload at one setting. + assert_eq!(results.lines().count(), 2, "{results}"); +} diff --git a/crates/devtools/tests/measurement/machine_and_tables.rs b/crates/devtools/tests/measurement/machine_and_tables.rs new file mode 100644 index 000000000..7b753e6dc --- /dev/null +++ b/crates/devtools/tests/measurement/machine_and_tables.rs @@ -0,0 +1,428 @@ +//! §2.5: the machine key, which tables a run reads, and the fallback to +//! the formulas. + +use std::path::Path; +use std::process::Command; + +use asap_accuracy::NoMeasuredGuarantees; +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::calibration_tables::{ + load_planning_tables, resolve_table_paths, TableOverrides, TablePath, CALIBRATION_DIR_ENV, +}; +use asap_devtools::measurement::machine::MachineDescription; +use asap_planner::DeploymentCapabilities; + +use super::common::{exact, facts, plan, workload, TempDir}; + +const E5_MODEL: &str = "Intel(R) Xeon(R) CPU E5-2683 v3 @ 2.00GHz"; + +/// `/proc/cpuinfo` of `cores` logical cores of `model`. +fn cpuinfo(model: &str, cores: u32) -> String { + (0..cores) + .map(|core| format!("processor\t: {core}\nmodel name\t: {model}\n\n")) + .collect() +} + +fn meminfo(kib: u64) -> String { + format!("MemTotal: {kib} kB\nMemFree: 1000 kB\n") +} + +/// 56 logical cores of the E5-2683 v3 on x86_64 with `memory_kib` of memory, +/// DIMM types `dimm_types` (EDAC) and an optional override. +fn e5( + memory_kib: u64, + dimm_types: &[&str], + memory_type_override: Option<&str>, +) -> MachineDescription { + MachineDescription::from_proc( + "x86_64", + &cpuinfo(E5_MODEL, 56), + &meminfo(memory_kib), + dimm_types, + memory_type_override, + "Linux 5.15", + ) + .unwrap() +} + +const E5_MEMORY_KIB: u64 = 263_724_000; +const E5_KEY: &str = "x86_64-intel-xeon-e5-2683-v3-56c-registered-ddr4"; + +/// Architecture, model, cores and memory type give the slug: `(R)`, `CPU` +/// and `@ 2.00GHz` are dropped from the model, the rest is lowercased and +/// dash-joined. +// decided 2026-11-16: key = architecture + model + logical cores + memory type +#[test] +fn machine_key_from_cpu_cores_memory() { + let machine = e5(E5_MEMORY_KIB, &["Registered-DDR4", "Registered-DDR4"], None); + assert_eq!(machine.cpu_architecture, "x86_64"); + assert_eq!(machine.logical_cores, 56); + assert_eq!(machine.memory_type, "Registered-DDR4"); + assert_eq!(machine.memory_kib, E5_MEMORY_KIB); + assert_eq!(machine.machine_key(), E5_KEY); +} + +/// The key is a function of architecture, CPU model, cores and memory type +/// only: this machine's detected key contains no token of its hostname that +/// the CPU model does not also contain, and the description has no hostname +/// to change. +#[test] +fn machine_key_ignores_hostname() { + let fixture = e5(E5_MEMORY_KIB, &["Registered-DDR4"], None).machine_key(); + for hostname in ["a", "xeon-e5-2683", "node-17"] { + // The fixture's inputs do not include a hostname, so every + // hostname gives the same key, and it is never the hostname. + assert_ne!(fixture, hostname); + } + let detected = MachineDescription::detect(None).unwrap(); + let key = detected.machine_key(); + let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default(); + let model = detected.cpu_model.to_lowercase(); + for token in hostname + .trim() + .to_lowercase() + .split(|c: char| !c.is_ascii_alphanumeric()) + { + if token.len() >= 3 && !model.contains(token) { + assert!( + !key.contains(token), + "key {key} contains hostname token {token}" + ); + } + } +} + +/// Total memory is not part of the key: 64 GiB and 256 GiB of the same +/// DIMM type give the same key, and `MemTotal` jitter across boots too. +#[test] +fn machine_key_ignores_memory_size() { + let dimms = ["Registered-DDR4"]; + let key = e5(E5_MEMORY_KIB, &dimms, None).machine_key(); + assert_eq!(key, E5_KEY); + assert_eq!(e5(64 * 1024 * 1024, &dimms, None).machine_key(), key); + assert_eq!(e5(256 * 1024 * 1024, &dimms, None).machine_key(), key); + assert_eq!( + e5(E5_MEMORY_KIB * 1005 / 1000, &dimms, None).machine_key(), + key + ); +} + +/// A VM without EDAC and without an override: `unknown-memory` ends the +/// key, and the warning names it and the override. +#[test] +fn machine_key_without_edac_is_unknown_memory() { + let machine = e5(E5_MEMORY_KIB, &[], None); + assert_eq!( + machine.machine_key(), + "x86_64-intel-xeon-e5-2683-v3-56c-unknown-memory" + ); + let warning = machine.warning().expect("a warning for unknown memory"); + assert!(warning.contains("unknown-memory"), "{warning}"); + assert!(warning.contains("ASAP_MEMORY_TYPE"), "{warning}"); + assert!(e5(E5_MEMORY_KIB, &["Registered-DDR4"], None) + .warning() + .is_none()); +} + +/// `--memory-type` / `ASAP_MEMORY_TYPE` supplies the type when EDAC has +/// none, and no warning is given. +#[test] +fn machine_key_memory_type_override() { + let machine = e5(E5_MEMORY_KIB, &[], Some("Registered-DDR4")); + assert_eq!(machine.machine_key(), E5_KEY); + assert!(machine.warning().is_none()); +} + +/// DIMMs of different types: the sorted unique values joined, whatever the +/// DIMM order. +#[test] +fn machine_key_mixed_dimm_types() { + for dimms in [ + ["Registered-DDR4", "LRDIMM-DDR4", "Registered-DDR4"], + ["LRDIMM-DDR4", "Registered-DDR4", "LRDIMM-DDR4"], + ] { + assert_eq!( + e5(E5_MEMORY_KIB, &dimms, None).machine_key(), + "x86_64-intel-xeon-e5-2683-v3-56c-lrdimm-ddr4-registered-ddr4" + ); + } +} + +/// Only `[a-z0-9-]`, no leading, trailing or double dashes. +#[test] +fn machine_key_slug_charset() { + let machine = MachineDescription::from_proc( + "aarch64", + &cpuinfo("AMD EPYC(TM) 7452 32-Core Processor @ 2.35GHz (R)", 8), + &meminfo(16_000_000), + &["Registered-DDR4"], + None, + "Linux", + ) + .unwrap(); + let key = machine.machine_key(); + assert!( + key.chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-'), + "{key}" + ); + assert!( + !key.contains("--") && !key.starts_with('-') && !key.ends_with('-'), + "{key}" + ); +} + +const KEY: &str = E5_KEY; + +/// `--costs` first, then `ASAP_CALIBRATION_DIR`, then `/calibration`, +/// then nothing (the formulas). +// provisional answer 15: an `ASAP_CALIBRATION_DIR` without the machine's +// file falls back to the formulas; the repo default is read only when the +// variable is unset. +#[test] +fn path_precedence() { + let env = TempDir::new("env"); + let repo = TempDir::new("repo"); + let given = env.write("given/costs.json", "{}"); + let env_costs = env.write(&format!("{KEY}/costs.json"), "{}"); + let env_accuracy = env.write(&format!("{KEY}/accuracy.json"), "{}"); + let repo_costs = repo.write(&format!("calibration/{KEY}/costs.json"), "{}"); + let overrides = TableOverrides { + costs: Some(given.clone()), + ..TableOverrides::default() + }; + let resolved = resolve_table_paths(&overrides, Some(&env.0), Some(&repo.0), KEY); + assert_eq!(resolved.costs, TablePath::Given(given)); + assert_eq!(resolved.accuracy, TablePath::Calibrated(env_accuracy)); + let resolved = + resolve_table_paths(&TableOverrides::default(), Some(&env.0), Some(&repo.0), KEY); + assert_eq!(resolved.costs, TablePath::Calibrated(env_costs)); + let resolved = resolve_table_paths(&TableOverrides::default(), None, Some(&repo.0), KEY); + assert_eq!(resolved.costs, TablePath::Calibrated(repo_costs)); + assert_eq!( + resolved.accuracy, + TablePath::Missing { + looked_at: repo.0.join(format!("calibration/{KEY}/accuracy.json")) + } + ); + let empty = TempDir::new("empty"); + let resolved = resolve_table_paths( + &TableOverrides::default(), + Some(&empty.0), + Some(&repo.0), + KEY, + ); + assert_eq!( + resolved.costs, + TablePath::Missing { + looked_at: empty.0.join(format!("{KEY}/costs.json")) + } + ); +} + +/// A small workload whose plan the tables can change. +fn rate_workload() -> asap_planner::PlanningWorkload { + workload( + &[("sum by (job) (rate(requests_total[5m]))", exact())], + 10_000, + &[("requests_total", facts(100, 1_000))], + ) +} + +/// No `costs.json` for this machine: one warning naming the machine key +/// and `asap_run calibrate`, and the plan is the default cost inputs' +/// plan, byte for byte. +#[test] +fn missing_costs_warns_and_falls_back() { + let dir = TempDir::new("missing"); + let resolved = resolve_table_paths(&TableOverrides::default(), Some(&dir.0), None, KEY); + let tables = load_planning_tables(&resolved, &TableOverrides::default(), KEY).unwrap(); + let costs_warnings: Vec<_> = tables + .warnings + .iter() + .filter(|warning| warning.contains("costs")) + .collect(); + assert_eq!(costs_warnings.len(), 1, "{:?}", tables.warnings); + assert!(costs_warnings[0].contains(KEY), "{costs_warnings:?}"); + assert!( + costs_warnings[0].contains("asap_run calibrate"), + "{costs_warnings:?}" + ); + let caps = DeploymentCapabilities::default(); + let fallback = plan( + &rate_workload(), + &tables.cost_inputs, + &tables.accuracy, + &caps, + ); + let default = plan( + &rate_workload(), + &CostInputs::default(), + &NoMeasuredGuarantees, + &caps, + ); + assert_eq!( + serde_json::to_string(&fallback.0).unwrap(), + serde_json::to_string(&default.0).unwrap() + ); +} + +fn scan_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: 20.0, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }], + ..MeasuredOperationCosts::default() + } +} + +/// Costs present, accuracy absent: exactly one warning, for the accuracy +/// table, and the plan is the costs-only plan with no measured accuracy. +#[test] +fn missing_accuracy_warns_separately() { + let dir = TempDir::new("costs_only"); + dir.write( + &format!("{KEY}/costs.json"), + &serde_json::to_string(&scan_table()).unwrap(), + ); + let resolved = resolve_table_paths(&TableOverrides::default(), Some(&dir.0), None, KEY); + let tables = load_planning_tables(&resolved, &TableOverrides::default(), KEY).unwrap(); + assert_eq!(tables.warnings.len(), 1, "{:?}", tables.warnings); + assert!( + tables.warnings[0].contains("accuracy"), + "{:?}", + tables.warnings + ); + assert_eq!(tables.cost_inputs.measured_operation_costs, scan_table()); + assert!(tables.accuracy.entries.is_empty()); + let costs_only = CostInputs { + measured_operation_costs: scan_table(), + ..CostInputs::default() + }; + let caps = DeploymentCapabilities::default(); + assert_eq!( + plan( + &rate_workload(), + &tables.cost_inputs, + &tables.accuracy, + &caps + ), + plan(&rate_workload(), &costs_only, &NoMeasuredGuarantees, &caps), + ); +} + +/// `--costs` wins over the table inside `--cost-inputs`, with a warning. +// provisional answer 15 +#[test] +fn costs_flag_wins_over_cost_inputs_table() { + let dir = TempDir::new("precedence"); + let inner = MeasuredOperationCosts { + ns_per_evaluation: 1_000.0, + ..scan_table() + }; + let cost_inputs = dir.write( + "cost_inputs.json", + &serde_json::to_string(&CostInputs { + measured_operation_costs: inner, + ..CostInputs::default() + }) + .unwrap(), + ); + let costs = dir.write("costs.json", &serde_json::to_string(&scan_table()).unwrap()); + let overrides = TableOverrides { + costs: Some(costs), + cost_inputs: Some(cost_inputs), + ..TableOverrides::default() + }; + let resolved = resolve_table_paths(&overrides, Some(&dir.0), None, KEY); + let tables = load_planning_tables(&resolved, &overrides, KEY).unwrap(); + assert_eq!(tables.cost_inputs.measured_operation_costs, scan_table()); + assert!( + tables + .warnings + .iter() + .any(|warning| warning.contains("--costs")), + "{:?}", + tables.warnings + ); +} + +/// A truncated or invalid table is an error naming its path, never a +/// silent fallback. +// provisional answer 15 +#[test] +fn corrupt_costs_is_an_error() { + for (name, contents) in [ + ( + "truncated", + r#"{"operators": [{"operator": "Scan", "ns_per_unit""#, + ), + ( + "negative", + r#"{"operators": [{"operator": "Scan", "ns_per_unit": -5.0}]}"#, + ), + ] { + let dir = TempDir::new(name); + let path = dir.write(&format!("{KEY}/costs.json"), contents); + let resolved = resolve_table_paths(&TableOverrides::default(), Some(&dir.0), None, KEY); + let error = load_planning_tables(&resolved, &TableOverrides::default(), KEY).unwrap_err(); + assert_eq!(error.path, path, "{name}"); + } +} + +/// A run of `bin` on `workload_path` with the calibration directory `dir`. +fn run_with_calibration_dir(bin: &str, args: &[&str], dir: &Path) -> String { + let output = Command::new(bin) + .args(args) + .env(CALIBRATION_DIR_ENV, dir) + .output() + .unwrap(); + String::from_utf8_lossy(&output.stderr).into_owned() +} + +/// `asap-planner` and `asap_run` resolve the same tables: with the same +/// empty calibration directory, both print the same missing-table +/// warnings. +#[test] +fn asap_planner_cli_uses_same_resolution() { + let dir = TempDir::new("cli"); + let workload = dir.write("w.json", &serde_json::to_string(&rate_workload()).unwrap()); + let workload = workload.to_str().unwrap(); + let warnings = |stderr: String| -> Vec { + stderr + .lines() + .filter(|line| line.contains("asap_run calibrate")) + .map(str::to_owned) + .collect() + }; + let planner = warnings(run_with_calibration_dir( + env!("CARGO_BIN_EXE_asap_planner"), + &["--workload", workload], + &dir.0, + )); + let run = warnings(run_with_calibration_dir( + env!("CARGO_BIN_EXE_asap_run"), + &[ + "execute", + "--workload", + workload, + "--data", + "facts", + "--at", + "600000", + ], + &dir.0, + )); + assert!( + !planner.is_empty(), + "asap-planner printed no missing-table warning" + ); + assert_eq!(planner, run); +} diff --git a/crates/devtools/tests/measurement/measured_accuracy.rs b/crates/devtools/tests/measurement/measured_accuracy.rs new file mode 100644 index 000000000..bf2d71f8c --- /dev/null +++ b/crates/devtools/tests/measurement/measured_accuracy.rs @@ -0,0 +1,342 @@ +//! §2.4: the accuracy calibration's aggregation, and the measured table +//! read by Pass 2 through the planner. + +use std::collections::BTreeSet; + +use asap_accuracy::{ + ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy, NoMeasuredGuarantees, +}; +use asap_cost::CostInputs; +use asap_devtools::measurement::accuracy_calibration::{ + error_quantile, worst_over_merge_counts, MergeCountErrors, +}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; +use asap_planner::{DeploymentCapabilities, PlanningModels, PlanningWorkload}; +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +use super::common::{facts, per_value, plan, workload}; + +fn point(failure_probability: f64, error: f64) -> ErrorAtFailureProbability { + ErrorAtFailureProbability { + failure_probability, + error, + } +} + +/// Rank errors of KLL k = 547 at δ = 0.01 after 1, 6 and 360 merges: +/// 0.0015, 0.0018, 0.0017. The table records the worst, 0.0018, not the +/// mean (0.001667) or the last (0.0017). +#[test] +fn worst_over_merge_counts_is_recorded() { + let after = |merge_count, error| MergeCountErrors { + sketch: SketchParams::Kll { k: 547 }, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric: AccuracyMetric::Rank, + merge_count, + trials: 1_000, + error_at: vec![point(0.01, error)], + }; + let entries = + worst_over_merge_counts(&[after(1, 0.0015), after(6, 0.0018), after(360, 0.0017)]); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].error_at, vec![point(0.01, 0.0018)]); + assert_eq!(entries[0].trials, 1_000); +} + +/// Errors i / 1e5 for i = 1..1000, given in descending order: the +/// nearest-rank 0.99 quantile is the ⌈0.99 · 1000⌉ = 990th smallest, +/// 0.0099. +// provisional answer 14 +#[test] +fn quantile_of_errors_nearest_rank() { + let errors: Vec = (1..=1000).rev().map(|i| i as f64 / 1e5).collect(); + assert_eq!(error_quantile(&errors, 0.01), 990.0 / 1e5); +} + +/// `quantile_over_time(0.99, m[5m])` every 10 s over 8 series sampled each +/// second, within rank error 0.0025 per value at δ = 0.01. +fn kll_workload() -> PlanningWorkload { + workload( + &[( + "quantile_over_time(0.99, m[5m])", + per_value("rank", 0.0025, 0.01), + )], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +/// Every KLL `k` Pass 2 connects to a readout through an edge that meets +/// its requirement. +fn admissible_kll_sizes(workload: &PlanningWorkload, models: PlanningModels<'_>) -> BTreeSet { + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_planning_stages_over( + lower_promql_roots(workload).unwrap(), + &entries, + &data, + models, + ) + .unwrap(); + let logical = run.run.logical_asap_candidates.expect("Pass 2 ran"); + logical + .edges + .iter() + .filter(|edge| edge.satisfies_logical_requirement) + .filter(|edge| { + matches!( + logical + .memo + .candidate(edge.consumer_candidate_id) + .expression + .operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + ) + }) + .filter_map(|edge| { + match logical + .memo + .candidate(edge.producer_candidate_id) + .logical_properties + .parameters() + { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + } + }) + .collect() +} + +/// The KLL `k` of the selected plan's sketch, if it has one. +fn selected_kll_size( + workload: &PlanningWorkload, + evidence: &dyn asap_accuracy::AccuracyEvidence, +) -> Option { + let (installable, _) = plan( + workload, + &CostInputs::default(), + evidence, + &DeploymentCapabilities::default(), + ); + installable + .dag + .nodes + .iter() + .find_map(|node| match node.parameters.as_ref() { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + }) +} + +/// Without a table, the closed form sizes KLL: of the predefined sizes, +/// k = 547 (rank error 0.004998 at δ = 0.01) misses 0.0025 and the smallest +/// admissible is k = 1404 (0.00200). +#[test] +fn kll_sized_by_closed_form_without_measured_table() { + let sizes = admissible_kll_sizes(&kll_workload(), PlanningModels::builtin()); + assert!(!sizes.contains(&547), "{sizes:?}"); + assert_eq!(sizes.first(), Some(&1_404), "{sizes:?}"); +} + +/// The fixture table: KLL k = 547 for the 0.99 quantile measured at rank +/// error 0.0024 at δ = 0.01 (within 0.0025). +fn kll_547_measured(metric: AccuracyMetric, k: u32) -> MeasuredAccuracyEntry { + MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k }, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric: metric, + trials: 1_000, + error_at: vec![ + point(0.003, 0.0031), + point(0.01, 0.0024), + point(0.05, 0.0016), + ], + } +} + +/// With the table, k = 547 is admissible and, its state being smaller than +/// k = 1404's, selected. +#[test] +fn measured_accuracy_shrinks_sketch() { + let table = MeasuredSketchAccuracy { + entries: vec![kll_547_measured(AccuracyMetric::Rank, 547)], + }; + let sizes = admissible_kll_sizes(&kll_workload(), PlanningModels::new(&table)); + assert!(sizes.contains(&547), "{sizes:?}"); + assert_eq!(selected_kll_size(&kll_workload(), &table), Some(547)); +} + +/// Entries for k = 547 and k = 1404 in the relative metric (not KLL's +/// rank) neither admit k = 547 nor prune k = 1404: the plan is the plan +/// without a table. +// provisional answer 7 +#[test] +fn wrong_metric_entry_does_not_prune() { + let table = MeasuredSketchAccuracy { + entries: vec![ + kll_547_measured(AccuracyMetric::Relative, 547), + kll_547_measured(AccuracyMetric::Relative, 1_404), + ], + }; + let with = admissible_kll_sizes(&kll_workload(), PlanningModels::new(&table)); + let without = admissible_kll_sizes(&kll_workload(), PlanningModels::builtin()); + assert_eq!(with, without); + assert!(with.contains(&1_404), "{with:?}"); + let caps = DeploymentCapabilities::default(); + assert_eq!( + plan(&kll_workload(), &CostInputs::default(), &table, &caps), + plan( + &kll_workload(), + &CostInputs::default(), + &NoMeasuredGuarantees, + &caps + ), + ); +} + +/// An empty table plans exactly as no measurement: the same plan, cost +/// report and pruned candidates (I7). +#[test] +fn empty_table_equals_no_measured() { + let workload = kll_workload(); + let empty = MeasuredSketchAccuracy::default(); + let caps = DeploymentCapabilities::default(); + assert_eq!( + plan(&workload, &CostInputs::default(), &empty, &caps), + plan( + &workload, + &CostInputs::default(), + &NoMeasuredGuarantees, + &caps + ), + ); + let pruned = |models: PlanningModels<'_>| { + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_planning_stages_over( + lower_promql_roots(&workload).unwrap(), + &entries, + &data, + models, + ) + .unwrap(); + format!( + "{:?} {:?}", + run.run + .logical_asap_candidates + .map(|logical| logical.pruned), + run.run + .physical_asap_candidates + .map(|physical| physical.pruned_candidates) + ) + }; + assert_eq!( + pruned(PlanningModels::new(&empty)), + pruned(PlanningModels::builtin()) + ); +} + +/// A small calibration's accuracy table holds I4: per entry, δ ascending +/// in (0, 1] and ≥ 3 / trials, errors finite, non-negative and +/// non-increasing in δ, and each the worst of the per-merge-count results +/// at that δ. +// provisional answer 13 +#[test] +fn accuracy_table_is_monotone() { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: true, + seed: 7, + }) + .unwrap(); + calibration.accuracy.validate().unwrap(); + for entry in &calibration.accuracy.entries { + let floor = 3.0 / entry.trials as f64; + for pair in entry.error_at.windows(2) { + assert!( + pair[0].failure_probability < pair[1].failure_probability, + "{entry:?}" + ); + assert!(pair[1].error <= pair[0].error, "{entry:?}"); + } + for at in &entry.error_at { + assert!( + at.failure_probability >= floor && at.failure_probability <= 1.0, + "{entry:?}" + ); + assert!(at.error.is_finite() && at.error >= 0.0, "{entry:?}"); + let worst = calibration + .accuracy_by_merge_count + .iter() + .filter(|raw| { + raw.sketch == entry.sketch + && raw.statistic == entry.statistic + && raw.accuracy_metric == entry.accuracy_metric + }) + .flat_map(|raw| raw.error_at.iter()) + .filter(|raw| raw.failure_probability == at.failure_probability) + .map(|raw| raw.error) + .fold(f64::NEG_INFINITY, f64::max); + assert_eq!(at.error, worst, "{entry:?}"); + } + } +} + +/// A full calibration's measurements lie within the sketches' known +/// bounds: DDSketch α = 0.01 never exceeds relative error 0.01 (a +/// deterministic bound); CMS (w, d) at δ ≥ e^−d plus a 99% binomial slack +/// has absolute error at most e / w of ‖f‖₁. KLL k = 547 at δ = 0.01 +/// within 1.2 × 0.004998 is a sanity check that only warns. +#[test] +#[ignore = "timing: run on a dedicated host"] +fn measured_within_known_bounds() { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: false, + seed: 7, + }) + .unwrap(); + for raw in &calibration.accuracy_by_merge_count { + match (&raw.sketch, raw.accuracy_metric) { + (SketchParams::DDSketch { alpha }, AccuracyMetric::Relative) if *alpha == 0.01 => { + for at in &raw.error_at { + assert!(at.error <= 0.01 * (1.0 + 1e-9), "{raw:?}"); + } + } + (SketchParams::Cms { width, depth }, metric) + if metric == AccuracyMetric::ABSOLUTE_STREAM_L1 => + { + let trials = raw.trials as f64; + let failure = (-(*depth as f64)).exp(); + // One-sided 99% slack on a binomial fraction of `trials`. + let slack = 2.326 * (failure * (1.0 - failure) / trials).sqrt(); + for at in raw + .error_at + .iter() + .filter(|at| at.failure_probability >= failure + slack) + { + assert!(at.error <= std::f64::consts::E / *width as f64, "{raw:?}"); + } + } + (SketchParams::Kll { k: 547 }, AccuracyMetric::Rank) => { + for at in raw + .error_at + .iter() + .filter(|at| at.failure_probability == 0.01) + { + if at.error > 1.2 * 0.004998 { + eprintln!( + "warning: KLL k = 547 measured {} > 1.2 x 0.004998", + at.error + ); + } + } + } + _ => {} + } + } +} diff --git a/crates/devtools/tests/measurement/mod.rs b/crates/devtools/tests/measurement/mod.rs new file mode 100644 index 000000000..30f4763a1 --- /dev/null +++ b/crates/devtools/tests/measurement/mod.rs @@ -0,0 +1,20 @@ +//! The `asap_run` test suite (`asap_devtools::measurement`), by the test +//! plan's sections. Tiers: T0 and T1 run in CI; T2 (timing) is ignored +//! with "timing: run on a dedicated host"; T3 (milestone scale) with +//! "milestone scale". Every expected value is computed by hand, from a +//! kernel or clock of known cost, from known sketch bounds or by +//! independent enumeration, never by the planner's own optimization or +//! cost code. A test that depends on a provisional answer of the test plan +//! says `// provisional answer N`. + +mod common; +mod cost_accounting; +mod cost_fit; +mod end_to_end; +mod execution; +mod gate; +mod machine_and_tables; +mod measured_accuracy; +mod oracle; +mod regression; +mod source_facts; diff --git a/crates/devtools/tests/measurement/oracle.rs b/crates/devtools/tests/measurement/oracle.rs new file mode 100644 index 000000000..6a07af362 --- /dev/null +++ b/crates/devtools/tests/measurement/oracle.rs @@ -0,0 +1,337 @@ +//! §2.11: the exhaustive oracle. Every plan over Stage 2's candidates is +//! enumerated independently of Stage 3 +//! (`asap_types::test_support::physical_plan_enumeration`), each is +//! measured, and the selected plan must be within τ of the cheapest +//! measured one, τ = 0.05 + twice the cheapest plan's coefficient of +//! variation over its repetitions. +// provisional answer 17 + +use std::collections::BTreeSet; + +use asap_devtools::measurement::benchmark::{ + bench_plan, installable_plan_of_selection, ArmMeasurement, BenchArm, BenchRequest, DataSetting, +}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest, PlanningTables}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::planning_stages::{ + lower_promql_roots, run_planning_stages_over, PlanningStagesRun, +}; +use asap_planner::{PlanningModels, PlanningWorkload}; +use asap_types::stages::PhysicalCandidateId; +use asap_types::test_support::physical_plan_enumeration::{ + every_physical_plan, PhysicalPlanChoice, +}; + +use super::common::{exact, facts, per_value, plan, workload}; + +const MOST_PLANS: usize = 1_000_000; + +fn exact_family() -> PlanningWorkload { + workload( + &[("sum by (job) (sum_over_time(m[5m]))", exact())], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +fn sketch_family() -> PlanningWorkload { + workload( + &[( + "quantile_over_time(0.99, m[5m])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +fn joint_sharing() -> PlanningWorkload { + workload( + &[ + ("sum by (job) (sum_over_time(m[5m]))", exact()), + ( + "quantile_over_time(0.99, m[5m])", + per_value("relative", 0.01, 0.01), + ), + ("count_over_time(m[5m])", exact()), + ], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +fn planning_run(workload: &PlanningWorkload, models: PlanningModels<'_>) -> PlanningStagesRun { + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + run_planning_stages_over( + lower_promql_roots(workload).unwrap(), + &entries, + &data, + models, + ) + .unwrap() +} + +/// Every enumerated plan is closed: each query root's group has a picked +/// candidate, one candidate per group, and each picked candidate's every +/// input has one admissible edge whose consumer is that candidate and +/// whose producer is the candidate picked for the producer's group. +#[test] +fn enumerated_plans_are_closed_under_edges() { + for workload in [exact_family(), sketch_family()] { + let run = planning_run(&workload, PlanningModels::builtin()); + let physical = run.run.physical_asap_candidates.expect("Stage 2 ran"); + let plans = every_physical_plan(&physical, MOST_PLANS).unwrap(); + assert!(!plans.is_empty()); + let distinct: BTreeSet<_> = plans.iter().collect(); + assert_eq!(distinct.len(), plans.len(), "a plan is enumerated twice"); + for plan in &plans { + for query in physical.logical_candidates.memo.queries() { + for group in query.root.operator_refs() { + assert!(plan.candidate_of_group.contains_key(group), "{plan:?}"); + } + } + for (&group, &candidate) in &plan.candidate_of_group { + assert_eq!( + physical.physical_candidates[candidate.0] + .logical_candidate_id + .group, + group + ); + for input in 0..physical.input_count(candidate) { + let edge_id = plan.edge_into_input[&(candidate, input)]; + assert!(physical + .admissible_edges_into(candidate, input) + .contains(&edge_id)); + let edge = physical.edge(edge_id); + assert_eq!(edge.consumer_candidate_id, candidate); + let producer_group = physical.physical_candidates[edge.producer_candidate_id.0] + .logical_candidate_id + .group; + assert_eq!( + plan.candidate_of_group[&producer_group], + edge.producer_candidate_id + ); + } + } + assert_eq!( + plan.edge_into_input.len(), + plan.candidate_of_group + .values() + .map(|&candidate| physical.input_count(candidate)) + .sum::() + ); + } + } +} + +/// The plan Stage 3 selects is one of the enumerated plans (validity): +/// its nodes' physical candidates are exactly one plan's candidates. +#[test] +fn enumerated_plans_contain_the_selected_plan() { + for workload in [exact_family(), sketch_family()] { + let run = planning_run(&workload, PlanningModels::builtin()); + let (dag, _) = run.run.selection.as_ref().expect("a plan is selected"); + let selected: BTreeSet = dag + .nodes + .iter() + .filter_map(|node| node.selected_physical_candidate) + .collect(); + let physical = run + .run + .physical_asap_candidates + .as_ref() + .expect("Stage 2 ran"); + let plans = every_physical_plan(physical, MOST_PLANS).unwrap(); + assert!( + plans.iter().any(|plan| plan.candidates() == selected), + "selected {selected:?} is none of the {} enumerated plans", + plans.len() + ); + } +} + +/// The joint workload's plans are few enough to measure each: at most +/// [`MOST_PLANS`]. Stage 2 gives it 452 physical candidates in 23 groups +/// (the exact family has 27 and 624 plans). +#[test] +fn joint_sharing_plan_space_is_enumerable() { + let run = planning_run(&joint_sharing(), PlanningModels::builtin()); + let physical = run.run.physical_asap_candidates.expect("Stage 2 ran"); + let plans = every_physical_plan(&physical, MOST_PLANS); + assert!(plans.is_ok(), "{plans:?}"); +} + +/// The calibrated tables of this machine. +fn calibrated_tables() -> PlanningTables { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: false, + seed: 7, + }) + .unwrap(); + PlanningTables { + cost_inputs: asap_cost::CostInputs { + measured_operation_costs: calibration.costs, + ..asap_cost::CostInputs::default() + }, + accuracy: calibration.accuracy, + warnings: Vec::new(), + } +} + +/// Every enumerated plan and the selected one, measured. +struct OracleRun { + enumerated: Vec<(PhysicalPlanChoice, ArmMeasurement)>, + selected: ArmMeasurement, +} + +fn run_oracle(workload: &PlanningWorkload) -> OracleRun { + let tables = calibrated_tables(); + let models = PlanningModels::new(&tables.accuracy).with_cost_inputs(&tables.cost_inputs); + let run = planning_run(workload, models); + let physical = run.run.physical_asap_candidates.expect("Stage 2 ran"); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let setting = DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms: 10_000, + raw_data_retained: true, + }; + let request = BenchRequest { + workload, + arms: vec![BenchArm::Planned], + settings: vec![setting.clone()], + series_count: 8, + duration_ms: 1_200_000, + repetitions: 5, + tables: tables.clone(), + seed: 1, + cost_gate: true, + output_root: None, + }; + let enumerated = every_physical_plan(&physical, MOST_PLANS) + .unwrap() + .into_iter() + .map(|choice| { + let installable = installable_plan_of_selection( + &physical, + &entries, + &choice.candidates(), + &choice.edges(), + ) + .unwrap(); + let measured = bench_plan(&installable, workload, &setting, &request).unwrap(); + (choice, measured) + }) + .collect(); + let (selected_plan, _) = plan( + workload, + &tables.cost_inputs, + &tables.accuracy, + &asap_planner::DeploymentCapabilities::default(), + ); + let selected = bench_plan(&selected_plan, workload, &setting, &request).unwrap(); + OracleRun { + enumerated, + selected, + } +} + +fn measured_cost(arm: &ArmMeasurement) -> f64 { + arm.measured_cost_per_second.expect("a measured cost") +} + +/// The selected plan's measured cost is within τ of the cheapest +/// enumerated plan's. +// provisional answer 23 +fn assert_within_tau(oracle: &OracleRun) { + let cheapest = oracle + .enumerated + .iter() + .map(|(_, arm)| arm) + .min_by(|a, b| measured_cost(a).total_cmp(&measured_cost(b))) + .expect("at least one plan"); + let repetitions = &cheapest.repetition_costs_per_second; + let mean = repetitions.iter().sum::() / repetitions.len() as f64; + let variance = repetitions + .iter() + .map(|cost| (cost - mean).powi(2)) + .sum::() + / repetitions.len() as f64; + let tau = 0.05 + 2.0 * variance.sqrt() / mean; + assert!( + measured_cost(&oracle.selected) <= (1.0 + tau) * measured_cost(cheapest), + "selected {} vs cheapest {} (tau {tau})", + measured_cost(&oracle.selected), + measured_cost(cheapest) + ); +} + +/// Spearman's rank correlation of two equally long samples, no ties +/// assumed. +fn spearman(left: &[f64], right: &[f64]) -> f64 { + let ranks = |values: &[f64]| { + let mut order: Vec = (0..values.len()).collect(); + order.sort_by(|&a, &b| values[a].total_cmp(&values[b])); + let mut ranks = vec![0.0; values.len()]; + for (rank, index) in order.into_iter().enumerate() { + ranks[index] = rank as f64; + } + ranks + }; + let (left, right) = (ranks(left), ranks(right)); + let n = left.len() as f64; + let d2: f64 = left.iter().zip(&right).map(|(a, b)| (a - b).powi(2)).sum(); + 1.0 - 6.0 * d2 / (n * (n * n - 1.0)) +} + +/// The exact family (exact from raw data, a query-time `SummaryAgg`, a +/// chain at each feasible width, no-raw variants): the selected plan is +/// within τ, and predicted and measured costs rank the plans alike +/// (Spearman ρ ≥ 0.8). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn oracle_exact_family() { + let oracle = run_oracle(&exact_family()); + assert_within_tau(&oracle); + let predicted: Vec = oracle + .enumerated + .iter() + .map(|(_, arm)| arm.predicted.cost_per_second.unwrap()) + .collect(); + let measured: Vec = oracle + .enumerated + .iter() + .map(|(_, arm)| measured_cost(arm)) + .collect(); + assert!(spearman(&predicted, &measured) >= 0.8); +} + +/// The sketch family (exact, DDSketch α ≤ 0.01 at query time or as a +/// chain; KLL excluded by the relative metric): every enumerated plan meets +/// the target, and the selected one is within τ. +// provisional answer 3 +#[test] +#[ignore = "timing: run on a dedicated host"] +fn oracle_sketch_family() { + let oracle = run_oracle(&sketch_family()); + for (choice, arm) in &oracle.enumerated { + assert!( + arm.measured_error.unwrap() <= 0.01 * (1.0 + 1e-9), + "{choice:?}" + ); + } + assert_within_tau(&oracle); +} + +/// The two families with `count_over_time(m[5m])`: the selected shared +/// plan is within τ, and the shared node is predicted once. +#[test] +#[ignore = "timing: run on a dedicated host"] +fn oracle_joint_sharing() { + let oracle = run_oracle(&joint_sharing()); + assert_within_tau(&oracle); + let nodes = &oracle.selected.predicted.predicted_usage.per_node; + let ids: BTreeSet<_> = nodes.iter().map(|node| node.node_id).collect(); + assert_eq!(ids.len(), nodes.len(), "a node is predicted twice"); +} diff --git a/crates/devtools/tests/measurement/regression.rs b/crates/devtools/tests/measurement/regression.rs new file mode 100644 index 000000000..c4bf33b4f --- /dev/null +++ b/crates/devtools/tests/measurement/regression.rs @@ -0,0 +1,186 @@ +//! §2.10: regression tests for the mis-rankings the feasibility run found, +//! with tables calibrated on the same machine. The small variants (100 +//! series, 70 minutes of data so a 1h window is covered) are T2; the full +//! ones (1,000 series) are T3. + +use asap_devtools::measurement::benchmark::{ + bench, bench_showcase, ArmMeasurement, BenchArm, BenchReport, BenchRequest, DataSetting, +}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest, PlanningTables}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_planner::PlanningWorkload; +use asap_types::ir::properties::ExecutionTiming; + +use super::common::{exact, facts, per_value, workload}; + +fn calibrated_tables() -> PlanningTables { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: false, + seed: 7, + }) + .unwrap(); + PlanningTables { + cost_inputs: asap_cost::CostInputs { + measured_operation_costs: calibration.costs, + ..asap_cost::CostInputs::default() + }, + accuracy: calibration.accuracy, + warnings: Vec::new(), + } +} + +fn arm(report: &BenchReport, which: BenchArm) -> &ArmMeasurement { + report + .arms + .iter() + .find(|arm| arm.arm == which) + .expect("the arm ran") +} + +fn cost(arm: &ArmMeasurement) -> f64 { + arm.measured_cost_per_second.expect("a measured cost") +} + +fn run(workload: &PlanningWorkload, series: u64, evaluation_interval_ms: u64) -> BenchReport { + bench(&BenchRequest { + workload, + arms: vec![BenchArm::Exact, BenchArm::Planned, BenchArm::NoRaw], + settings: vec![DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms, + raw_data_retained: true, + }], + series_count: series, + duration_ms: 70 * 60_000, + repetitions: 5, + tables: calibrated_tables(), + seed: 1, + cost_gate: true, + output_root: None, + }) + .unwrap() +} + +/// `sum by (job) (rate(requests_total[1h]))` at 1 Hz every 300 s with raw +/// data: today a query-time `SummaryAgg` (331 ms p50) is selected though +/// the stored chain is 45× faster, and predicted wrongly ranked. The +/// selected plan must cost at most 1.05 × the cheaper of exact and the +/// chain, and the predicted order of the two must be the measured one. +// provisional answer 1 and provisional answer 20 +fn rate_every_300s(series: u64) { + let workload = workload( + &[("sum by (job) (rate(requests_total[1h]))", exact())], + 300_000, + &[("requests_total", facts(series, 1_000))], + ); + let report = run(&workload, series, 300_000); + let (exact_arm, planned, chain) = ( + arm(&report, BenchArm::Exact), + arm(&report, BenchArm::Planned), + arm(&report, BenchArm::NoRaw), + ); + assert!(cost(planned) <= 1.05 * cost(exact_arm).min(cost(chain))); + let predicted = |arm: &ArmMeasurement| arm.predicted.cost_per_second.unwrap(); + assert_eq!( + predicted(exact_arm) < predicted(chain), + cost(exact_arm) < cost(chain), + "predicted and measured disagree on query time against the chain" + ); +} + +#[test] +#[ignore = "timing: run on a dedicated host"] +fn regression_rate_1hz_every_300s() { + rate_every_300s(100); +} + +#[test] +#[ignore = "milestone scale"] +fn regression_rate_1hz_every_300s_full() { + rate_every_300s(1_000); +} + +/// `quantile_over_time(0.99, latency[1h])` at 1 Hz every 10 s, 1% relative +/// at δ = 0.01, raw data: today a DDSketch built from raw data at query +/// time runs at 0.50× exact. The selected plan must cost at most 1.05 × +/// exact (speedup ≥ 0.95×). +// provisional answer 1 +fn quantile_not_slower(series: u64) { + let workload = workload( + &[( + "quantile_over_time(0.99, latency[1h])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("latency", facts(series, 1_000))], + ); + let report = run(&workload, series, 10_000); + let (exact_arm, planned) = ( + arm(&report, BenchArm::Exact), + arm(&report, BenchArm::Planned), + ); + assert!(cost(planned) <= 1.05 * cost(exact_arm)); + let speedup = exact_arm.measured.as_ref().unwrap().p50_query_ms + / planned.measured.as_ref().unwrap().p50_query_ms; + assert!(speedup >= 0.95, "speedup {speedup}"); +} + +#[test] +#[ignore = "timing: run on a dedicated host"] +fn regression_quantile_half_speed() { + quantile_not_slower(100); +} + +#[test] +#[ignore = "milestone scale"] +fn regression_quantile_half_speed_full() { + quantile_not_slower(1_000); +} + +/// Over the full showcase, the planner ranks the selected plan against +/// exact as measured in at least 90% of the rows, and measured / predicted +/// stays within 1.5× per plan kind (by the timing of the plan's first +/// root). +#[test] +#[ignore = "milestone scale"] +fn regression_ranking_rate() { + let report = bench_showcase(false, &calibrated_tables(), None).unwrap(); + let mut right = 0; + let mut rows = 0; + for exact_arm in report.arms.iter().filter(|arm| arm.arm == BenchArm::Exact) { + let Some(planned) = report.arms.iter().find(|arm| { + arm.arm == BenchArm::Planned + && arm.setting == exact_arm.setting + && arm.plan.query_texts == exact_arm.plan.query_texts + }) else { + continue; + }; + rows += 1; + let predicted_order = + planned.predicted.cost_per_second < exact_arm.predicted.cost_per_second; + let measured_order = cost(planned) < cost(exact_arm); + right += usize::from(predicted_order == measured_order); + } + assert!(rows > 0); + assert!( + right as f64 >= 0.9 * rows as f64, + "{right} of {rows} ranked right" + ); + for kind in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] { + for arm in report.arms.iter().filter(|arm| { + let root = arm.plan.dag.roots[0]; + arm.plan + .dag + .nodes + .iter() + .any(|node| node.id == root && node.output_state.timing == kind) + }) { + let ratio = cost(arm) / arm.predicted.cost_per_second.unwrap(); + assert!( + (1.0 / 1.5..=1.5).contains(&ratio), + "{kind:?}: measured / predicted {ratio}" + ); + } + } +} diff --git a/crates/devtools/tests/measurement/source_facts.rs b/crates/devtools/tests/measurement/source_facts.rs new file mode 100644 index 000000000..2e03d1ef7 --- /dev/null +++ b/crates/devtools/tests/measurement/source_facts.rs @@ -0,0 +1,447 @@ +//! §2.6: source facts from hand-made data. +//! +//! Source `m`: +//! +//! | Series | Samples (t ms : value) | +//! |---|---| +//! | A `{job=a,instance=1}` | 0:1, 1000:2, 2000:−3, 4000:0 | +//! | B `{job=a,instance=2}` | 0:5, 1000:10 | +//! | C `{job=b,instance=1}` | 500:0.5, 1500:0.25, 2500:100 | +//! | D `{job=b}` (no instance) | 0:7, 2000:7 | +//! +//! Source `lat`: two series, every value in [0.001, 3], one of them −0.0. + +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; +use asap_devtools::measurement::samples::{read_samples_jsonl, samples_of_generated, Sample}; +use asap_devtools::measurement::source_facts::{ + observe_source_statistics, source_facts_of_samples, FactsOptions, ObservedMagnitude, + SuggestedFact, +}; +use asap_planner::PlanningWorkload; +use asap_types::stages::ValueSign; +use asap_types::workload::LabelSetDistinctValues; + +use super::common::{per_value, sample, whole_result, workload}; + +fn fixture() -> Vec { + let a = [("job", "a"), ("instance", "1")]; + let b = [("job", "a"), ("instance", "2")]; + let c = [("job", "b"), ("instance", "1")]; + let d = [("job", "b")]; + let mut samples = vec![ + sample("m", &a, 0, 1.0), + sample("m", &a, 1_000, 2.0), + sample("m", &a, 2_000, -3.0), + sample("m", &a, 4_000, 0.0), + sample("m", &b, 0, 5.0), + sample("m", &b, 1_000, 10.0), + sample("m", &c, 500, 0.5), + sample("m", &c, 1_500, 0.25), + sample("m", &c, 2_500, 100.0), + sample("m", &d, 0, 7.0), + sample("m", &d, 2_000, 7.0), + ]; + let x = [("host", "x")]; + let y = [("host", "y")]; + samples.extend([ + sample("lat", &x, 0, 0.001), + sample("lat", &x, 1_000, 3.0), + sample("lat", &y, 0, -0.0), + sample("lat", &y, 1_000, 1.5), + ]); + samples +} + +const QUERY: &str = "sum by (job, instance) (sum_over_time(m[2s]))"; +const LAT_QUERY: &str = "max_over_time(lat[2s])"; + +/// The fixture's workload, with per-value targets: no fact splits δ, so +/// the label counts are cost-only. +fn per_value_workload() -> PlanningWorkload { + workload( + &[ + (QUERY, per_value("absolute", 1.0, 0.01)), + (LAT_QUERY, per_value("absolute", 1.0, 0.01)), + ], + 1_000, + &[], + ) +} + +/// The same with a whole-result target: the series and label counts bound +/// G, over which δ splits, so they become guarantee facts. +fn whole_result_workload() -> PlanningWorkload { + workload( + &[ + (QUERY, whole_result("absolute", 1.0, 0.01)), + (LAT_QUERY, per_value("absolute", 1.0, 0.01)), + ], + 1_000, + &[], + ) +} + +fn observed_m() -> asap_devtools::measurement::source_facts::ObservedSourceStatistics { + observe_source_statistics(&fixture(), &per_value_workload()) + .remove("m") + .unwrap() +} + +/// Four distinct label sets. +#[test] +fn facts_series_count() { + assert_eq!(observed_m().series_count, 4); +} + +/// Gaps A 1000, 1000, 2000; B 1000; C 1000, 1000; D 2000. Per-series +/// medians 1000, 1000, 1000, 2000, whose median is 1000 (the mean gap, +/// 1286, is wrong). A second fixture separates the two medians: P has 10 +/// samples 1 s apart, Q and R 2 samples 5 s apart; the pooled median gap +/// is 1000, the median of per-series medians 5000. +// provisional answer 19 +#[test] +fn facts_sample_interval_median_gap() { + assert_eq!(observed_m().sample_interval_ms, Some(1_000)); + let mut samples: Vec = (0..10) + .map(|i| sample("s", &[("id", "p")], i * 1_000, 1.0)) + .collect(); + for id in ["q", "r"] { + samples.push(sample("s", &[("id", id)], 0, 1.0)); + samples.push(sample("s", &[("id", id)], 5_000, 1.0)); + } + let workload = workload( + &[("max_over_time(s[10s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&samples, &workload) + .remove("s") + .unwrap(); + assert_eq!(observed.sample_interval_ms, Some(5_000)); +} + +/// `job` takes 2 values; `instance` 3 (1, 2 and missing on D). +#[test] +fn facts_distinct_per_label() { + let observed = observed_m(); + assert_eq!(observed.distinct_values_per_label["job"], 2); + assert_eq!(observed.distinct_values_per_label["instance"], 3); +} + +/// `by (job, instance)` sees 4 combinations (a/1, a/2, b/1, b/missing), +/// not the product 6; `lat`'s query groups by no label set of two, so it +/// gets no entry. +// provisional answer 19 +#[test] +fn facts_label_set_from_workload_by() { + let observed = observe_source_statistics(&fixture(), &per_value_workload()); + let mut sets = observed["m"].distinct_values_per_label_set.clone(); + for set in &mut sets { + set.labels.sort(); + } + assert_eq!( + sets, + vec![LabelSetDistinctValues { + labels: vec!["instance".into(), "job".into()], + distinct_values: 4, + }] + ); + assert!(observed["lat"].distinct_values_per_label_set.is_empty()); +} + +/// Zeros are excluded: `m.value` spans 0.25 to 100 (a min_abs of 0 would +/// be invalid). +#[test] +fn facts_column_magnitudes_exclude_zero() { + assert_eq!( + observed_m().column_magnitudes["value"], + ObservedMagnitude { + min_abs: 0.25, + max_abs: 100.0, + } + ); + let zeros = vec![sample("z", &[], 0, 0.0), sample("z", &[], 1_000, -0.0)]; + let workload = workload( + &[("max_over_time(z[2s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&zeros, &workload) + .remove("z") + .unwrap(); + assert!(observed.column_magnitudes.get("value").is_none()); +} + +/// Signs by sign bit: `m` has −3, so no sign suggestion; `lat` holds +/// −0.0, so no `non_negative` either (a `v >= 0.0` check would accept it). +#[test] +fn facts_sign_suggestion_sign_bit() { + let report = + source_facts_of_samples(&fixture(), &per_value_workload(), FactsOptions::default()) + .unwrap(); + assert_eq!( + report.observed["m"].column_signs["value"], + ValueSign::Unknown + ); + assert_ne!( + report.observed["lat"].column_signs["value"], + ValueSign::NonNegative + ); + assert!( + !report.suggestions.iter().any(|suggestion| matches!( + &suggestion.fact, + SuggestedFact::ColumnSign { + sign: ValueSign::NonNegative, + .. + } + )), + "{:?}", + report.suggestions + ); +} + +/// With per-value targets, the cost-only facts are written as observed: +/// series count 4, interval 1000, label counts 2 and 3, the label set's 4; +/// the magnitudes with their margin (0.125 to 200). +// provisional answer 5 and provisional answer 6 +#[test] +fn facts_cost_only_written_directly() { + let report = + source_facts_of_samples(&fixture(), &per_value_workload(), FactsOptions::default()) + .unwrap(); + let m = &report.data_workload.sources["m"]; + assert_eq!(m.series_count, Some(4)); + assert_eq!(m.sample_interval.map(|interval| interval.0), Some(1_000)); + assert_eq!(m.distinct_values_per_label["job"], 2); + assert_eq!(m.distinct_values_per_label["instance"], 3); + assert_eq!(m.distinct_values_per_label_set[0].distinct_values, 4); + assert_eq!( + report.column_magnitudes["m"]["value"], + ObservedMagnitude { + min_abs: 0.125, + max_abs: 200.0, + } + ); +} + +/// With a whole-result target, the guarantee facts are suggested with a +/// ×2 margin, ⌈2×⌉ for counts: instance 6, job 4, series 8, the label set +/// 8; distinct values per 2 s window at most 2 observed, so 4; norms over +/// one group's 2 s window at most ‖f‖₁ = 2 and ‖f‖₂ = √2 observed, so 4 +/// and 2√2. They are written only with `--trust-observed-bounds`. +// provisional answer 5 and provisional answer 19 +#[test] +fn facts_guarantee_margin() { + let suggested = source_facts_of_samples( + &fixture(), + &whole_result_workload(), + FactsOptions::default(), + ) + .unwrap(); + let observed = &suggested.observed["m"]; + assert_eq!(observed.max_distinct_per_evaluation, Some(2)); + let norms = observed.stream_norms.unwrap(); + assert_eq!(norms.l1, Some(2.0)); + assert!((norms.l2.unwrap() - 2f64.sqrt()).abs() < 1e-12); + let m_facts: Vec<&SuggestedFact> = suggested + .suggestions + .iter() + .filter(|suggestion| suggestion.source == "m") + .map(|suggestion| &suggestion.fact) + .collect(); + let has = |fact: &SuggestedFact| m_facts.contains(&fact); + assert!( + has(&SuggestedFact::DistinctValuesPerLabel { + label: "instance".into(), + distinct_values: 6 + }), + "{m_facts:?}" + ); + assert!( + has(&SuggestedFact::DistinctValuesPerLabel { + label: "job".into(), + distinct_values: 4 + }), + "{m_facts:?}" + ); + assert!(has(&SuggestedFact::SeriesCount(8)), "{m_facts:?}"); + assert!( + has(&SuggestedFact::MaxDistinctPerEvaluation(4)), + "{m_facts:?}" + ); + assert!( + m_facts + .iter() + .any(|fact| matches!(fact, SuggestedFact::StreamNorms(norms) + if norms.l1 == Some(4.0) && (norms.l2.unwrap() - 2.0 * 2f64.sqrt()).abs() < 1e-12)), + "{m_facts:?}" + ); + // Not written by default. + let m = &suggested.data_workload.sources["m"]; + assert_eq!(m.series_count, None); + assert!(m.distinct_values_per_label.is_empty()); + assert_eq!(m.max_distinct_per_evaluation, None); + assert_eq!(m.stream_norms, None); + // Written when trusted. + let trusted = source_facts_of_samples( + &fixture(), + &whole_result_workload(), + FactsOptions { + trust_observed_bounds: true, + }, + ) + .unwrap(); + let m = &trusted.data_workload.sources["m"]; + assert_eq!(m.series_count, Some(8)); + assert_eq!(m.distinct_values_per_label["instance"], 6); + assert_eq!(m.max_distinct_per_evaluation, Some(4)); +} + +/// After the margins, the label set's count (⌈2 · 4⌉ = 8) stays at most +/// the product of its labels' counts (4 · 6 = 24), and the written +/// workload validates. +#[test] +fn facts_margined_set_stays_valid() { + let trusted = source_facts_of_samples( + &fixture(), + &whole_result_workload(), + FactsOptions { + trust_observed_bounds: true, + }, + ) + .unwrap(); + let m = &trusted.data_workload.sources["m"]; + for set in &m.distinct_values_per_label_set { + let product: u64 = set + .labels + .iter() + .map(|label| m.distinct_values_per_label[label]) + .product(); + assert!(set.distinct_values <= product, "{set:?}"); + } + let mut written = whole_result_workload(); + written.data_workload = Some(trusted.data_workload.clone()); + written.validate().unwrap(); +} + +/// A counter 0, 5, 10, 2, 7 resets once: still non-negative; magnitudes +/// over the raw samples (zeros excluded): 2 to 10. +#[test] +fn facts_counter_reset() { + let samples: Vec = [0.0, 5.0, 10.0, 2.0, 7.0] + .iter() + .enumerate() + .map(|(i, &value)| sample("c_total", &[], i as i64 * 1_000, value)) + .collect(); + let workload = workload( + &[("rate(c_total[5s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&samples, &workload) + .remove("c_total") + .unwrap(); + assert_eq!(observed.column_signs["value"], ValueSign::NonNegative); + assert_eq!( + observed.column_magnitudes["value"], + ObservedMagnitude { + min_abs: 2.0, + max_abs: 10.0, + } + ); +} + +/// A series of one sample has no gap and does not panic: the interval +/// comes from the other series. +#[test] +fn facts_single_sample_series() { + let samples = vec![ + sample("s", &[("id", "lonely")], 0, 1.0), + sample("s", &[("id", "p")], 0, 1.0), + sample("s", &[("id", "p")], 3_000, 1.0), + ]; + let workload = workload( + &[("max_over_time(s[10s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&samples, &workload) + .remove("s") + .unwrap(); + assert_eq!(observed.sample_interval_ms, Some(3_000)); + let alone = vec![sample("t", &[], 0, 1.0)]; + let workload = workload_over("t"); + assert_eq!( + observe_source_statistics(&alone, &workload)["t"].sample_interval_ms, + None + ); +} + +fn workload_over(metric: &str) -> PlanningWorkload { + workload( + &[( + &format!("max_over_time({metric}[10s])"), + per_value("absolute", 1.0, 0.01), + )], + 1_000, + &[], + ) +} + +/// The JSONL schema: one `{labels, ts_ms, value}` object per line. +// provisional answer 18 +#[test] +fn facts_reads_jsonl_samples() { + let text = "{\"labels\": {\"__name__\": \"m\", \"job\": \"a\"}, \"ts_ms\": 1000, \"value\": 2.0}\n\ + {\"labels\": {\"__name__\": \"m\", \"job\": \"b\"}, \"ts_ms\": 2000, \"value\": -0.5}\n"; + let samples = read_samples_jsonl(text).unwrap(); + assert_eq!( + samples, + vec![ + sample("m", &[("job", "a")], 1_000, 2.0), + sample("m", &[("job", "b")], 2_000, -0.5), + ] + ); + let error = read_samples_jsonl("{\"labels\": {}, \"ts_ms\": 1}\n").unwrap_err(); + assert_eq!(error.line, 1); +} + +/// Generated data round-trips through facts: 50 series over `host` (100 +/// values) × `endpoint` (8 values) every 10 s give series count 50, +/// interval 10,000, at most 50 hosts and 8 endpoints. +#[test] +fn facts_datagen_round_trip() { + let description = TimeSeriesDescription { + labels: vec![ + SeriesLabelSpec { + name: "host".into(), + cardinality: 100, + skewness: 0.0, + }, + SeriesLabelSpec { + name: "endpoint".into(), + cardinality: 8, + skewness: 1.1, + }, + ], + series_num: 50, + start_ms: 0, + sample_interval_ms: 10_000, + duration_ms: 600_000, + value_distribution: DataDistribution::Uniform(UniformParameter { + lower_bound: 0.0, + upper_bound: 1.0, + seed: 0, + }), + counter: false, + seed: 42, + }; + let samples = samples_of_generated("g", &description.generate().unwrap()).unwrap(); + let observed = observe_source_statistics(&samples, &workload_over("g")) + .remove("g") + .unwrap(); + assert_eq!(observed.series_count, 50); + assert_eq!(observed.sample_interval_ms, Some(10_000)); + assert!(observed.distinct_values_per_label["host"] <= 50); + assert!(observed.distinct_values_per_label["endpoint"] <= 8); +} diff --git a/crates/types/src/test_support/mod.rs b/crates/types/src/test_support/mod.rs index c6343f5d7..695f07b76 100644 --- a/crates/types/src/test_support/mod.rs +++ b/crates/types/src/test_support/mod.rs @@ -9,3 +9,6 @@ pub mod forwarders; pub mod memo_builders; /// Operator expressions the memo builders and tests share. pub mod memo_fixtures; +/// Every plan over Stage 2's physical candidates, for the exhaustive +/// oracle. +pub mod physical_plan_enumeration; diff --git a/crates/types/src/test_support/physical_plan_enumeration.rs b/crates/types/src/test_support/physical_plan_enumeration.rs new file mode 100644 index 000000000..3bafa6b0e --- /dev/null +++ b/crates/types/src/test_support/physical_plan_enumeration.rs @@ -0,0 +1,169 @@ +//! Every plan over Stage 2's physical candidates +//! ([`PhysicalASAPMemoCandidates`]), by plain enumeration, for an +//! exhaustive oracle that checks plan selection against every +//! alternative. It is written from Stage 2's output alone and shares no +//! code with Stage 3. +//! +//! A plan picks one physical candidate for each memo group it reaches, +//! starting at the query roots' groups, and one admissible edge into each +//! input of each picked candidate, whose producer is the candidate picked +//! for the producer's group. Example: `sum_over_time(m[5m])` with an +//! exact query-time candidate and an ingestion-time chain candidate in the +//! root group, each reading the scan's single candidate, has two plans. +//! +//! One candidate per group: a plan that reads one group through two of +//! its candidates (say exact for one query and a sketch for another) is +//! not enumerated. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; + +/// One plan: the candidate picked per memo group, and the edge feeding +/// each input of each picked candidate. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct PhysicalPlanChoice { + pub candidate_of_group: BTreeMap, + /// Keyed by (consumer, input index of the consumer). + pub edge_into_input: BTreeMap<(PhysicalCandidateId, usize), PhysicalEdgeId>, +} + +impl PhysicalPlanChoice { + /// The picked candidates. + pub fn candidates(&self) -> BTreeSet { + self.candidate_of_group.values().copied().collect() + } + + /// The picked edges. + pub fn edges(&self) -> BTreeSet { + self.edge_into_input.values().copied().collect() + } +} + +/// The enumeration stopped: more than `limit` plans. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TooManyPlans { + pub limit: usize, +} + +/// What is left to pick in a partial plan. +#[derive(Debug, Clone, Copy)] +enum Pick { + /// A candidate for this group (a query root's). + Group(MemoGroupId), + /// An edge into input `.1` of candidate `.0`. + Input(PhysicalCandidateId, usize), +} + +/// Every plan over `physical`, at most `limit` of them. +pub fn every_physical_plan( + physical: &PhysicalASAPMemoCandidates, + limit: usize, +) -> Result, TooManyPlans> { + let mut root_groups = Vec::new(); + for query in physical.logical_candidates.memo.queries() { + for &group in query.root.operator_refs() { + if !root_groups.contains(&group) { + root_groups.push(group); + } + } + } + let mut candidates_in_group: BTreeMap> = BTreeMap::new(); + for (index, candidate) in physical.physical_candidates.iter().enumerate() { + candidates_in_group + .entry(candidate.logical_candidate_id.group) + .or_default() + .push(PhysicalCandidateId(index)); + } + let mut plans = Vec::new(); + let pending: Vec = root_groups.into_iter().rev().map(Pick::Group).collect(); + extend( + physical, + &candidates_in_group, + &mut PhysicalPlanChoice::default(), + pending, + &mut plans, + limit, + )?; + Ok(plans) +} + +/// The group of physical candidate `candidate`. +fn group_of(physical: &PhysicalASAPMemoCandidates, candidate: PhysicalCandidateId) -> MemoGroupId { + physical.physical_candidates[candidate.0] + .logical_candidate_id + .group +} + +/// `pending` with a pick for every input of `candidate` on top. +fn with_inputs_of( + physical: &PhysicalASAPMemoCandidates, + candidate: PhysicalCandidateId, + pending: &[Pick], +) -> Vec { + let mut next = pending.to_vec(); + next.extend( + (0..physical.input_count(candidate)) + .rev() + .map(|input| Pick::Input(candidate, input)), + ); + next +} + +/// Complete `choice` by every way of making the `pending` picks (the +/// last one first), adding each complete plan to `plans`. +fn extend( + physical: &PhysicalASAPMemoCandidates, + candidates_in_group: &BTreeMap>, + choice: &mut PhysicalPlanChoice, + mut pending: Vec, + plans: &mut Vec, + limit: usize, +) -> Result<(), TooManyPlans> { + let Some(pick) = pending.pop() else { + if plans.len() == limit { + return Err(TooManyPlans { limit }); + } + plans.push(choice.clone()); + return Ok(()); + }; + match pick { + Pick::Group(group) => { + if choice.candidate_of_group.contains_key(&group) { + return extend(physical, candidates_in_group, choice, pending, plans, limit); + } + for &candidate in candidates_in_group.get(&group).into_iter().flatten() { + choice.candidate_of_group.insert(group, candidate); + let next = with_inputs_of(physical, candidate, &pending); + extend(physical, candidates_in_group, choice, next, plans, limit)?; + choice.candidate_of_group.remove(&group); + } + Ok(()) + } + Pick::Input(consumer, input) => { + for &edge_id in physical.admissible_edges_into(consumer, input) { + let producer = physical.edge(edge_id).producer_candidate_id; + let group = group_of(physical, producer); + let newly_picked = match choice.candidate_of_group.get(&group) { + Some(&picked) if picked != producer => continue, + Some(_) => false, + None => true, + }; + choice.edge_into_input.insert((consumer, input), edge_id); + let next = if newly_picked { + choice.candidate_of_group.insert(group, producer); + with_inputs_of(physical, producer, &pending) + } else { + pending.clone() + }; + let result = extend(physical, candidates_in_group, choice, next, plans, limit); + if newly_picked { + choice.candidate_of_group.remove(&group); + } + choice.edge_into_input.remove(&(consumer, input)); + result?; + } + Ok(()) + } + } +}