From 165e433327f64952cc74ef61e49d523f82783c5f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:33:42 +0000 Subject: [PATCH 1/2] test(devtools): asap_run test suite (tests and API skeleton only) The test suite of the asap_run test plan: cost fit, measured cost lookup, measured accuracy lookup and its Stage 1 hook, machine key and table resolution, source facts, the execution driver, cost accounting, the bench gate, regressions, the exhaustive oracle, end to end and determinism. T2 tests are ignored as timing tests, T3 as milestone scale. Non-test code is the API skeleton only: the asap_run bin and the devtools measurement module, every body todo!; the MeasuredSketchAccuracy data types in asap-accuracy, their validate and AccuracyEvidence impl todo!; the aqpbm-datagen git dependency in devtools. The oracle's plan enumerator over Stage 2's candidates is test support in asap-types, written independently of Stage 3. --- Cargo.lock | 33 ++ crates/accuracy/Cargo.toml | 1 + crates/accuracy/src/lib.rs | 8 +- crates/accuracy/src/measured.rs | 97 ++++ .../tests/measured_sketch_accuracy.rs | 245 ++++++++++ crates/cost/src/node/usage/tests.rs | 144 ++++++ crates/cost/tests/measured_operation_costs.rs | 245 ++++++++++ crates/devtools/Cargo.toml | 4 + crates/devtools/src/bin/asap_run.rs | 20 + crates/devtools/src/lib.rs | 4 +- .../src/measurement/accuracy_calibration.rs | 39 ++ crates/devtools/src/measurement/benchmark.rs | 215 +++++++++ .../src/measurement/calibration_tables.rs | 130 +++++ .../src/measurement/cost_calibration.rs | 165 +++++++ crates/devtools/src/measurement/execution.rs | 140 ++++++ crates/devtools/src/measurement/machine.rs | 36 ++ crates/devtools/src/measurement/mod.rs | 33 ++ crates/devtools/src/measurement/samples.rs | 79 ++++ .../devtools/src/measurement/source_facts.rs | 108 +++++ crates/devtools/tests/main.rs | 1 + crates/devtools/tests/measurement/common.rs | 200 ++++++++ .../tests/measurement/cost_accounting.rs | 431 +++++++++++++++++ crates/devtools/tests/measurement/cost_fit.rs | 239 ++++++++++ .../devtools/tests/measurement/end_to_end.rs | 349 ++++++++++++++ .../devtools/tests/measurement/execution.rs | 373 +++++++++++++++ crates/devtools/tests/measurement/gate.rs | 298 ++++++++++++ .../tests/measurement/machine_and_tables.rs | 367 ++++++++++++++ .../tests/measurement/measured_accuracy.rs | 342 ++++++++++++++ crates/devtools/tests/measurement/mod.rs | 20 + crates/devtools/tests/measurement/oracle.rs | 337 +++++++++++++ .../devtools/tests/measurement/regression.rs | 186 ++++++++ .../tests/measurement/source_facts.rs | 447 ++++++++++++++++++ crates/types/src/test_support/mod.rs | 3 + .../test_support/physical_plan_enumeration.rs | 169 +++++++ 34 files changed, 5506 insertions(+), 2 deletions(-) create mode 100644 crates/accuracy/src/measured.rs create mode 100644 crates/accuracy/tests/measured_sketch_accuracy.rs create mode 100644 crates/cost/tests/measured_operation_costs.rs create mode 100644 crates/devtools/src/bin/asap_run.rs create mode 100644 crates/devtools/src/measurement/accuracy_calibration.rs create mode 100644 crates/devtools/src/measurement/benchmark.rs create mode 100644 crates/devtools/src/measurement/calibration_tables.rs create mode 100644 crates/devtools/src/measurement/cost_calibration.rs create mode 100644 crates/devtools/src/measurement/execution.rs create mode 100644 crates/devtools/src/measurement/machine.rs create mode 100644 crates/devtools/src/measurement/mod.rs create mode 100644 crates/devtools/src/measurement/samples.rs create mode 100644 crates/devtools/src/measurement/source_facts.rs create mode 100644 crates/devtools/tests/measurement/common.rs create mode 100644 crates/devtools/tests/measurement/cost_accounting.rs create mode 100644 crates/devtools/tests/measurement/cost_fit.rs create mode 100644 crates/devtools/tests/measurement/end_to_end.rs create mode 100644 crates/devtools/tests/measurement/execution.rs create mode 100644 crates/devtools/tests/measurement/gate.rs create mode 100644 crates/devtools/tests/measurement/machine_and_tables.rs create mode 100644 crates/devtools/tests/measurement/measured_accuracy.rs create mode 100644 crates/devtools/tests/measurement/mod.rs create mode 100644 crates/devtools/tests/measurement/oracle.rs create mode 100644 crates/devtools/tests/measurement/regression.rs create mode 100644 crates/devtools/tests/measurement/source_facts.rs create mode 100644 crates/types/src/test_support/physical_plan_enumeration.rs diff --git a/Cargo.lock b/Cargo.lock index 84b535f2d..2c1d933af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,18 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "aqpbm-datagen" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?rev=cf8f86735e6f4c59d9827e3ae5d5bd28e8cd3ee1#cf8f86735e6f4c59d9827e3ae5d5bd28e8cd3ee1" +dependencies = [ + "rand 0.9.5", + "rand_distr", + "rand_xoshiro", + "serde", + "thiserror 2.0.18", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -286,6 +298,7 @@ version = "0.1.0" dependencies = [ "asap-types", "asap_sketchlib", + "serde", "serde_json", ] @@ -305,6 +318,7 @@ dependencies = [ name = "asap-devtools" version = "0.1.0" dependencies = [ + "aqpbm-datagen", "asap-accuracy", "asap-cost", "asap-executor", @@ -2580,6 +2594,25 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "recursive" version = "0.1.1" diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 666477f7e..7d2da01b9 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } +serde = { version = "1", features = ["derive"] } serde_json = "1" [dev-dependencies] diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index abf154e7d..72638069e 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -2,7 +2,8 @@ //! theoretical guarantee of reading a statistic from a summary with given //! parameters ([`theoretical_guarantee`], from the per-algorithm //! guarantees in `guarantee`), the deployment's measured accuracy -//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract +//! ([`AccuracyEvidence`], in `evidence`, and the measured table +//! [`MeasuredSketchAccuracy`], in `measured`), each operator's error contract //! ([`ErrorContract`], in `error_contract`), the error metrics of a memo //! and the metric rule ([`ErrorMetricsOfMemo`], in `metric_rule`, with the //! metric of a readout's error from `readout`), each readout's required @@ -19,6 +20,7 @@ mod error_contract; mod evidence; mod guarantee; mod logical_edge; +mod measured; mod metric_rule; mod readout; mod readout_caps; @@ -27,5 +29,9 @@ pub use error_contract::ErrorContract; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; pub use logical_edge::within_limit; +pub use measured::{ + ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy, + MeasuredSketchAccuracyError, +}; pub use metric_rule::ErrorMetricsOfMemo; pub use readout_caps::{readout_levels, ReadoutRequiredAccuracy}; diff --git a/crates/accuracy/src/measured.rs b/crates/accuracy/src/measured.rs new file mode 100644 index 000000000..6481dc64b --- /dev/null +++ b/crates/accuracy/src/measured.rs @@ -0,0 +1,97 @@ +//! The deployment's measured sketch accuracy: the errors a calibration run +//! measured on the executor's own sketches, which Pass 2 reads through +//! [`AccuracyEvidence`] before each summary's theoretical guarantee. +//! +//! Example: KLL with `k = 800`, read for `Quantile { q: 0.99 }`, measured +//! over 1,000 trials, has rank error 0.00176 at failure probability 0.01 +//! and 0.00121 at 0.05. A request at failure probability 0.02 gets +//! 0.00176 (the largest recorded failure probability at most 0.02); a +//! request at 0.001 gets no measured guarantee, so the theoretical one +//! applies. + +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchParams, SketchStatistic}; +use serde::{Deserialize, Serialize}; + +use crate::AccuracyEvidence; + +/// Errors measured on the executor's own sketches. Default: empty, every +/// sketch keeps its theoretical guarantee. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredSketchAccuracy { + pub entries: Vec, +} + +/// One sketch configuration, read for one statistic, in one metric. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MeasuredAccuracyEntry { + /// The exact parameters measured; only these match. + pub sketch: SketchParams, + /// The readout, e.g. `Quantile { q: 0.99 }`; only the same statistic + /// matches. + pub statistic: SketchStatistic, + /// The metric every recorded error is in, e.g. `Rank` for KLL. + pub accuracy_metric: AccuracyMetric, + /// Independent trials; every recorded failure probability is at least + /// 3 / trials. + pub trials: u32, + /// Ascending failure probability; the error does not increase as the + /// failure probability grows. + pub error_at: Vec, +} + +/// One measured point: the error not exceeded except with probability +/// `failure_probability`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErrorAtFailureProbability { + pub failure_probability: f64, + /// The (1 − δ) quantile of the per-value errors, the worst over the + /// measured merge counts, in the entry's `accuracy_metric`. + pub error: f64, +} + +/// Why a measured accuracy table is not valid. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum MeasuredSketchAccuracyError { + /// Entry `entry` has a failure probability outside (0, 1] or below + /// 3 / trials. + FailureProbabilityOutOfRange { + entry: usize, + failure_probability: f64, + }, + /// Entry `entry` has a negative or non-finite error. + InvalidError { entry: usize, error: f64 }, + /// Entry `entry`'s failure probabilities are not ascending, or its + /// error increases as the failure probability grows. + NotMonotone { entry: usize }, + /// Entries `first` and `second` measure the same sketch, statistic and + /// metric. + DuplicateEntry { first: usize, second: usize }, +} + +impl MeasuredSketchAccuracy { + /// Check every entry: failure probabilities in (0, 1] and at least + /// 3 / trials, errors finite and non-negative, ascending failure + /// probability with non-increasing error, and no two entries for one + /// (sketch, statistic, metric). + pub fn validate(&self) -> Result<(), MeasuredSketchAccuracyError> { + todo!("asap_run: implement after test approval") + } +} + +impl AccuracyEvidence for MeasuredSketchAccuracy { + fn measured_guarantee( + &self, + _summary_type: &FieldDataType, + _parameters: &SketchParams, + _statistic: &SketchStatistic, + _failure_probability: f64, + ) -> Option { + todo!("asap_run: implement after test approval") + } +} diff --git a/crates/accuracy/tests/measured_sketch_accuracy.rs b/crates/accuracy/tests/measured_sketch_accuracy.rs new file mode 100644 index 000000000..34eeccf9f --- /dev/null +++ b/crates/accuracy/tests/measured_sketch_accuracy.rs @@ -0,0 +1,245 @@ +//! The measured accuracy table as Pass 2's [`AccuracyEvidence`]: a request +//! at failure probability δ takes the largest recorded δ' ≤ δ of an entry +//! with exactly its parameters, statistic and metric, and nothing else. +//! Every expected value is read off the fixture by hand. + +use asap_accuracy::{ + theoretical_guarantee, AccuracyEvidence, ErrorAtFailureProbability, MeasuredAccuracyEntry, + MeasuredSketchAccuracy, +}; +use asap_types::accuracy::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; + +fn kll() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +fn p99() -> SketchStatistic { + SketchStatistic::Quantile { q: 0.99 } +} + +fn point(failure_probability: f64, error: f64) -> ErrorAtFailureProbability { + ErrorAtFailureProbability { + failure_probability, + error, + } +} + +/// KLL k = 547 read for the 0.99 quantile, rank error over 1,000 trials: +/// 0.0031 at δ = 0.003, 0.0024 at 0.01, 0.0016 at 0.05. +fn kll_547_entry() -> MeasuredAccuracyEntry { + MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k: 547 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Rank, + trials: 1_000, + error_at: vec![ + point(0.003, 0.0031), + point(0.01, 0.0024), + point(0.05, 0.0016), + ], + } +} + +fn table() -> MeasuredSketchAccuracy { + MeasuredSketchAccuracy { + entries: vec![kll_547_entry()], + } +} + +/// The error, failure probability and metric of a guarantee. +fn parts(guarantee: &AccuracyGuarantee) -> (f64, f64, AccuracyMetric) { + ( + guarantee.spec.error.evaluate().unwrap(), + guarantee.spec.failure_probability.evaluate().unwrap(), + guarantee.spec.accuracy_metric, + ) +} + +/// δ = 0.01 takes 0.01's error; 0.02 takes 0.01's too (the largest +/// recorded δ' ≤ 0.02, not the nearest 0.05 above); 0.05 and 1.0 (no +/// requirement) take 0.05's. +#[test] +fn lookup_takes_largest_recorded_delta_at_most_request() { + let table = table(); + let at = |delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + }; + assert_eq!(at(0.01), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.02), (0.0024, 0.01, AccuracyMetric::Rank)); + assert_eq!(at(0.05), (0.0016, 0.05, AccuracyMetric::Rank)); + assert_eq!(at(1.0), (0.0016, 0.05, AccuracyMetric::Rank)); +} + +/// δ = 0.002 is below every recorded δ': no measured guarantee, never +/// 0.0031 extrapolated downward. +#[test] +fn lookup_no_extrapolation_below_smallest() { + let guarantee = + table().measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002); + assert_eq!(guarantee, None); +} + +/// Where the table has no measurement (δ = 0.002 above), the theoretical +/// guarantee applies: KLL k = 547 has rank error 2.296 / 547^0.9723 = +/// 0.004998 at δ ≥ 0.01, scaled below 0.01 by z(1 − δ/2) / z(0.995): +/// 0.004998 · 3.0902 / 2.5758 = 0.005997 at δ = 0.002. +#[test] +fn closed_form_applies_below_the_measured_range() { + let guarantee = + theoretical_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.002).unwrap(); + let (error, _, metric) = parts(&guarantee); + assert_eq!(metric, AccuracyMetric::Rank); + assert!((error - 0.005997).abs() < 5e-6, "{error}"); +} + +/// Only the exact parameters, statistic and metric match: k = 548, the +/// median, or the relative metric get nothing. +#[test] +fn lookup_exact_params_statistic_metric() { + let table = table(); + assert_eq!( + table.measured_guarantee(&kll(), &SketchParams::Kll { k: 548 }, &p99(), 0.01), + None + ); + assert_eq!( + table.measured_guarantee( + &kll(), + &SketchParams::Kll { k: 547 }, + &SketchStatistic::Quantile { q: 0.5 }, + 0.01 + ), + None + ); + let relative = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + accuracy_metric: AccuracyMetric::Relative, + ..kll_547_entry() + }], + }; + // provisional answer 7: an entry in another metric than the readout's + // own (Rank for KLL) is never returned. + assert_eq!( + relative.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), 0.01), + None + ); +} + +/// An empty table measures nothing. +#[test] +fn empty_table_measures_nothing() { + let empty = MeasuredSketchAccuracy::default(); + for delta in [0.001, 0.01, 1.0] { + assert_eq!( + empty.measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta), + None + ); + } +} + +/// Lookup monotonicity (I10): a larger δ never gets a larger measured +/// error than a smaller one. +#[test] +fn lookup_error_does_not_grow_with_delta() { + let table = table(); + let deltas = [0.003, 0.005, 0.01, 0.02, 0.05, 0.5, 1.0]; + let errors: Vec = deltas + .iter() + .map(|&delta| { + parts( + &table + .measured_guarantee(&kll(), &SketchParams::Kll { k: 547 }, &p99(), delta) + .unwrap(), + ) + .0 + }) + .collect(); + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{errors:?}" + ); +} + +/// A deterministic sketch keeps its deterministic guarantee: a measured +/// (probabilistic) entry for DDSketch α = 0.01 is not returned. +// provisional answer 8 +#[test] +fn measured_entry_never_replaces_a_deterministic_guarantee() { + let table = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: SketchParams::DDSketch { alpha: 0.01 }, + statistic: p99(), + accuracy_metric: AccuracyMetric::Relative, + trials: 1_000, + error_at: vec![point(0.01, 0.004)], + }], + }; + let guarantee = table.measured_guarantee( + &FieldDataType::Sketch(SketchAlgorithm::DDSketch), + &SketchParams::DDSketch { alpha: 0.01 }, + &p99(), + 0.01, + ); + assert_eq!(guarantee, None); +} + +/// Each invalid table is rejected. +#[test] +fn table_validation() { + assert!(table().validate().is_ok()); + let with = |error_at: Vec| MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + error_at, + ..kll_547_entry() + }], + }; + let rejected = [ + // δ = 0.002 < 3 / 1,000 trials. + ("delta below 3/trials", with(vec![point(0.002, 0.004)])), + // The error grows with δ. + ( + "error increasing in delta", + with(vec![point(0.01, 0.002), point(0.05, 0.003)]), + ), + // δ not ascending. + ( + "unsorted deltas", + with(vec![point(0.05, 0.0016), point(0.01, 0.0024)]), + ), + ("delta zero", with(vec![point(0.0, 0.01)])), + ("delta above one", with(vec![point(1.5, 0.01)])), + ("negative error", with(vec![point(0.01, -1.0)])), + ("NaN error", with(vec![point(0.01, f64::NAN)])), + ( + "duplicate entry", + MeasuredSketchAccuracy { + entries: vec![kll_547_entry(), kll_547_entry()], + }, + ), + ]; + for (name, table) in rejected { + assert!(table.validate().is_err(), "{name} must be rejected"); + } +} + +/// The table's JSON is the issue's example format, and round-trips. +#[test] +fn accuracy_json_round_trips() { + let text = r#"{"entries": [{"sketch": {"Kll": {"k": 800}}, "statistic": {"Quantile": {"q": 0.99}}, + "accuracy_metric": "rank", "trials": 1000, "error_at": [ + {"failure_probability": 0.01, "error": 0.00176}, + {"failure_probability": 0.05, "error": 0.00121}]}]}"#; + let table: MeasuredSketchAccuracy = serde_json::from_str(text).unwrap(); + assert_eq!(table.entries[0].sketch, SketchParams::Kll { k: 800 }); + assert_eq!(table.entries[0].error_at[1], point(0.05, 0.00121)); + let written = serde_json::to_string(&table).unwrap(); + assert_eq!( + serde_json::from_str::(&written).unwrap(), + table + ); +} diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index efd664bf0..99bee20bf 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -2,6 +2,7 @@ use std::time::Duration; +use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::ExecutionDataState; use asap_types::ir::scalar::ColumnRef; use asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode; @@ -735,3 +736,146 @@ fn a_ddsketch_over_a_counter_declaring_its_value_unknown_keeps_both_stores() { ddsketch_costed_bytes(true, None) ); } + +/// A query-time producer of `rows` plain rows, for a node under test. +fn rows_of<'a>(producer: &'a PhysicalASAPDAGNode, rows: u64) -> ProducerOutput<'a> { + ProducerOutput { + payload: &producer.payload, + output_schema: &producer.output_schema, + output: EdgeStatistics { + rows, + bytes: rows * 16, + }, + sources: BTreeSet::new(), + family: StateFamily::None, + unchanged_columns: BTreeSet::new(), + tumbling_windows: 1, + chain_tumbling_window_width_ms: None, + } +} + +/// A one-minute `TimeRange` over node 0. +fn one_minute_range() -> PhysicalASAPDAGNode { + node(Payload::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: 0, + })) +} + +/// A query-time `SummaryAgg` of an exact sum over 600 input rows, with a +/// measured entry of 10 ns per row and 500 ns per computation, takes +/// 600 · 10 + 500 = 6,500 ns = 0.0065 ms per execution; computed once a +/// second, 6.5e-6 core-seconds per second, from the table. +#[test] +fn node_priced_by_hand() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::SummaryAgg, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 10.0, + ns_per_invocation: 500.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let build = node(Payload::ASAP(ASAPOp::SummaryAgg { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })); + let usage = node_resource_usage_and_cost( + &build, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.0065).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); + assert!( + (usage.cost.cpu_core_seconds_per_second - 6.5e-6).abs() < 1e-15, + "{}", + usage.cost.cpu_core_seconds_per_second + ); +} + +/// An `Aggregate` is priced from the table only when each of its measures +/// has an entry: with an entry for `sum` (10 ns per row) and none for +/// `quantile`, `Aggregate[sum, quantile]` keeps the formulas, and +/// `Aggregate[sum]` over 600 rows takes 600 · 10 = 6,000 ns. +#[test] +fn aggregate_needs_every_measure() { + let mut settings = CostInputs::default(); + settings.resource_speed.ms_per_cpu_op = 1e-6; + settings + .measured_operation_costs + .operators + .push(crate::OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("sum".into()), + ns_per_unit: 10.0, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }); + let data = DataWorkload::default(); + let range = one_minute_range(); + let aggregate = |measures: Vec| { + node(Payload::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::none()), + output_names: vec![String::new(); measures.len()], + filters: vec![None; measures.len()], + measures, + having: None, + child: 0, + })) + }; + let both = aggregate(vec![ + AggIntent::Sum { col: None }, + AggIntent::Quantile { col: None, q: 0.5 }, + ]); + let usage = node_resource_usage_and_cost( + &both, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Analytical); + let sum = aggregate(vec![AggIntent::Sum { col: None }]); + let usage = node_resource_usage_and_cost( + &sum, + &[rows_of(&range, 600)], + (1, 1.0), + None, + &data, + &settings, + ) + .unwrap(); + assert_eq!(usage.cost.cost_source, NodeCostSource::Measured); + assert!( + (usage.work_ms_per_execution - 0.006).abs() < 1e-12, + "{}", + usage.work_ms_per_execution + ); +} diff --git a/crates/cost/tests/measured_operation_costs.rs b/crates/cost/tests/measured_operation_costs.rs new file mode 100644 index 000000000..2a91ddd93 --- /dev/null +++ b/crates/cost/tests/measured_operation_costs.rs @@ -0,0 +1,245 @@ +//! The measured cost table: [`lookup`] (exact match, interpolation in log +//! size, phase shadowing, no extrapolation), its JSON round trip and the +//! formats the `asap_run` design deletes. Every expected price is +//! computed by hand from the entries below. + +use asap_cost::{ + lookup, CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind, StateFamily, +}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::summary_type::{ExactKind, SketchParams}; + +/// A sketch's size for interpolation: KLL's `k`, 1 for any other sketch. +fn size_of(params: &SketchParams) -> f64 { + match params { + SketchParams::Kll { k } => *k as f64, + _ => 1.0, + } +} + +/// A `SummaryAgg` entry for KLL with `k` at `phase`. +fn kll_entry( + k: u32, + phase: Option, + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +) -> OperatorCost { + OperatorCost { + operator: OperatorKind::SummaryAgg, + phase, + sketch: Some(SketchParams::Kll { k }), + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation, + memory_bytes_per_state, + } +} + +/// KLL `k = 200` at 100 ns/unit, 1,000 ns/invocation, 2,000 bytes, and +/// `k = 800` at 300 ns/unit, 3,000 ns/invocation, 8,000 bytes, for both +/// phases. +fn kll_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, Some(8_000.0)), + ], + ..MeasuredOperationCosts::default() + } +} + +/// A price's fields: ns per unit, ns per invocation, bytes per state. +#[derive(Debug, PartialEq)] +struct Price { + ns_per_unit: f64, + ns_per_invocation: f64, + memory_bytes_per_state: Option, +} + +fn kll_price_at(table: &MeasuredOperationCosts, phase: ExecutionTiming, k: u32) -> Option { + lookup( + table, + OperatorKind::SummaryAgg, + phase, + StateFamily::Sketch(&SketchParams::Kll { k }), + size_of, + ) + .map(|price| Price { + ns_per_unit: price.ns_per_unit, + ns_per_invocation: price.ns_per_invocation, + memory_bytes_per_state: price.memory_bytes_per_state, + }) +} + +fn assert_close(actual: f64, expected: f64) { + assert!( + (actual - expected).abs() <= 1e-9 * expected.abs().max(1.0), + "{actual} != {expected}" + ); +} + +/// An entry with exactly the node's parameters is its price, not an +/// interpolation. +#[test] +fn lookup_exact_params_win() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 200).unwrap(); + assert_eq!(price.ns_per_unit, 100.0); + assert_eq!(price.ns_per_invocation, 1_000.0); + assert_eq!(price.memory_bytes_per_state, Some(2_000.0)); +} + +/// k = 400 lies at ln(400/200) / ln(800/200) = 0.5 between the two +/// entries in log size: 100 + 0.5·200 = 200 ns/unit, 2,000 ns/invocation, +/// 5,000 bytes. Linear in size would give 166.7 ns/unit. +#[test] +fn lookup_interpolates_in_log_size() { + let price = kll_price_at(&kll_table(), ExecutionTiming::QueryTime, 400).unwrap(); + assert_close(price.ns_per_unit, 200.0); + assert_close(price.ns_per_invocation, 2_000.0); + assert_close(price.memory_bytes_per_state.unwrap(), 5_000.0); +} + +/// Below the smallest and above the largest measured size, the nearest +/// entry's price: 100 at k = 100 (not 33), 300 at k = 1600 (not 367). +#[test] +fn lookup_no_extrapolation() { + let table = kll_table(); + let below = kll_price_at(&table, ExecutionTiming::QueryTime, 100).unwrap(); + let above = kll_price_at(&table, ExecutionTiming::QueryTime, 1_600).unwrap(); + assert_eq!(below.ns_per_unit, 100.0); + assert_eq!(above.ns_per_unit, 300.0); +} + +/// A query-time entry for k = 200 at 999 ns/unit hides both both-phase +/// entries at query time: k = 400 has only k = 200 to go by, so 999. At +/// ingestion time there is no ingestion entry, so the both-phase pair +/// interpolates to 200. +#[test] +fn lookup_phase_entries_shadow_both() { + let mut table = kll_table(); + table.operators.push(kll_entry( + 200, + Some(ExecutionTiming::QueryTime), + 999.0, + 0.0, + None, + )); + let query = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + let ingestion = kll_price_at(&table, ExecutionTiming::IngestionTime, 400).unwrap(); + assert_eq!(query.ns_per_unit, 999.0); + assert_close(ingestion.ns_per_unit, 200.0); +} + +/// With no bytes on k = 800, k = 400 has one memory neighbour only: no +/// measured bytes (the formulas' bytes apply), never an interpolation +/// with 0. +#[test] +fn lookup_memory_needs_both_neighbours() { + let table = MeasuredOperationCosts { + operators: vec![ + kll_entry(200, None, 100.0, 1_000.0, Some(2_000.0)), + kll_entry(800, None, 300.0, 3_000.0, None), + ], + ..MeasuredOperationCosts::default() + }; + let price = kll_price_at(&table, ExecutionTiming::QueryTime, 400).unwrap(); + assert_eq!(price.memory_bytes_per_state, None); +} + +/// KLL entries do not price a DDSketch node. +#[test] +fn lookup_other_algorithm_misses() { + let price = lookup( + &kll_table(), + OperatorKind::SummaryAgg, + ExecutionTiming::QueryTime, + StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), + size_of, + ); + assert!(price.is_none()); +} + +/// A table with one of everything a JSON file can hold. +fn full_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![ + kll_entry( + 200, + Some(ExecutionTiming::IngestionTime), + 100.5, + 1_000.25, + Some(2_048.0), + ), + OperatorCost { + operator: OperatorKind::SummaryMerge, + phase: Some(ExecutionTiming::QueryTime), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 3.75, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Aggregate, + phase: None, + sketch: None, + exact: None, + measure: Some("quantile".into()), + ns_per_unit: 12.0, + ns_per_invocation: 250.0, + memory_bytes_per_state: None, + }, + OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: 20.0, + ns_per_invocation: 5_000.0, + memory_bytes_per_state: None, + }, + ], + ns_per_evaluation: 50_000.0, + ..MeasuredOperationCosts::default() + } +} + +/// `costs.json` round-trips: written, read back and written again it is +/// byte-equal, and it validates. +#[test] +fn costs_json_round_trips() { + let table = full_table(); + let written = serde_json::to_string_pretty(&table).unwrap(); + let read: MeasuredOperationCosts = serde_json::from_str(&written).unwrap(); + assert_eq!(read, table); + assert_eq!(serde_json::to_string_pretty(&read).unwrap(), written); + let inputs = CostInputs { + measured_operation_costs: read, + ..CostInputs::default() + }; + inputs.validate().unwrap(); +} + +/// The design deletes `provenance` (machine information lives in +/// `machine.json`) and the array format of earlier versions: a table +/// carrying either no longer loads. +// provisional answer 22 +#[test] +fn costs_json_rejects_removed_fields() { + let with_provenance = r#"{"operators": [], "ns_per_evaluation": 0.0, + "provenance": {"executor_version": "0.1.0", "machine": "x86_64"}}"#; + assert!( + serde_json::from_str::(with_provenance).is_err(), + "a table with `provenance` must not load" + ); + let legacy_array = r#"[{"sketch": {"Kll": {"k": 200}}, "operation": "update", + "cpu_ns_per_operation": 100.0}]"#; + assert!( + serde_json::from_str::(legacy_array).is_err(), + "a table in the array format must not load" + ); +} diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 80079502b..321b083d1 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -33,6 +33,10 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } +# The data `asap_run` ingests: sketch-bench's synthetic time-series +# generator, pinned to the commit that makes it a standalone library. Only +# the developer tools depend on it. +aqpbm-datagen = { git = "https://github.com/ProjectASAP/sketch-bench", rev = "cf8f86735e6f4c59d9827e3ae5d5bd28e8cd3ee1" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/devtools/src/bin/asap_run.rs b/crates/devtools/src/bin/asap_run.rs new file mode 100644 index 000000000..8057c742d --- /dev/null +++ b/crates/devtools/src/bin/asap_run.rs @@ -0,0 +1,20 @@ +//! `asap_run`: execute plans on the Planner's executor, calibrate measured +//! unit costs and sketch accuracy per machine, bench predicted against +//! measured cost, and compute source facts from data +//! (`asap_devtools::measurement`). +//! +//! ```text +//! asap_run execute --workload w.json [--data facts|file.jsonl] --at [,...] +//! asap_run calibrate [--families all|] [--small] +//! asap_run bench --workload w.json --arms exact,planned,noraw --duration +//! asap_run bench --suite showcase [--small] [--no-cost-gate] +//! asap_run facts --data --workload w.json [--trust-observed-bounds] +//! ``` +//! +//! Every subcommand reads the machine's tables as `asap-planner` does: +//! `--costs` / `--accuracy`, else `ASAP_CALIBRATION_DIR` or +//! `/calibration`, else the formulas with a warning. + +fn main() { + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 2b1e21f71..fcb859793 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -18,11 +18,13 @@ //! candidates for the viewer document. [`corpora`] splits the checked-in //! query corpora and holds their SQL catalogs. [`rqe_atomic_costs`] converts //! sketch-bench's measured per-operation costs into plan selection's -//! measured table. [`planner_fuzz`] is the metamorphic fuzz tester for +//! measured table. [`measurement`] runs selected plans on the executor +//! and measures them, behind the `asap_run` bin. [`planner_fuzz`] is the metamorphic fuzz tester for //! missed reuse and sharing behind the `planner_fuzz` bin. pub mod candidate_labels; pub mod corpora; +pub mod measurement; pub mod planner_fuzz; pub mod planner_layering_examples; pub mod planning_stages; diff --git a/crates/devtools/src/measurement/accuracy_calibration.rs b/crates/devtools/src/measurement/accuracy_calibration.rs new file mode 100644 index 000000000..e13b7cc01 --- /dev/null +++ b/crates/devtools/src/measurement/accuracy_calibration.rs @@ -0,0 +1,39 @@ +//! Accuracy calibration (`asap_run calibrate`): for each sketch +//! configuration × readout statistic × metric, seeded independent trials on +//! the executor's own sketches at each measured merge count, recorded as +//! the (1 − δ) quantile of the per-value errors, the worst over the merge +//! counts (`MeasuredSketchAccuracy`). +//! +//! Example: KLL `k = 547` read for `Quantile { q: 0.99 }` has rank errors +//! 0.0015, 0.0018 and 0.0017 at δ = 0.01 after 1, 6 and 360 merges; the +//! table records 0.0018. + +use asap_accuracy::{ErrorAtFailureProbability, MeasuredAccuracyEntry}; +use asap_types::accuracy::AccuracyMetric; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +/// The nearest-rank (1 − δ) quantile of `errors`: the ⌈(1 − δ)·n⌉-th +/// smallest. +pub fn error_quantile(errors: &[f64], failure_probability: f64) -> f64 { + let _ = (errors, failure_probability); + todo!("asap_run: implement after test approval") +} + +/// One configuration's errors after `merge_count` merges, as +/// `calibrate-run.json` records them. +#[derive(Debug, Clone, PartialEq)] +pub struct MergeCountErrors { + pub sketch: SketchParams, + pub statistic: SketchStatistic, + pub accuracy_metric: AccuracyMetric, + pub merge_count: u32, + pub trials: u32, + pub error_at: Vec, +} + +/// One table entry per (sketch, statistic, metric) of `results`: at each +/// failure probability, the worst error over the merge counts. +pub fn worst_over_merge_counts(results: &[MergeCountErrors]) -> Vec { + let _ = results; + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/benchmark.rs b/crates/devtools/src/measurement/benchmark.rs new file mode 100644 index 000000000..db2589c0d --- /dev/null +++ b/crates/devtools/src/measurement/benchmark.rs @@ -0,0 +1,215 @@ +//! Benchmarks (`asap_run bench`): each arm of a workload run end to end +//! over the same data, its measured usage and error beside the planner's +//! predicted cost, and the showcase gate. +//! +//! Gate example: an exact arm measured at cost 100 per second and a +//! selected plan at 105 pass (105 ≤ 1.05 × 100); 105.01 fails. A selected +//! plan whose measured error exceeds its target fails whatever its cost. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use asap_planner::{InstallablePlan, PlanningWorkload, SelectedPlanReport}; +use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; +use asap_types::workload::QueryWorkloadEntry; + +use crate::measurement::calibration_tables::PlanningTables; + +/// The plans one benchmark compares over the same data. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum BenchArm { + /// The exact plan: the reference every error is measured against. + Exact, + /// The plan selected with the resolved tables. + Planned, + /// The plan selected with `raw_data_retained: false`. + NoRaw, +} + +/// The data setting of one gate row, e.g. 1 Hz evaluated every 10 s with +/// raw data retained. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DataSetting { + pub sample_interval_ms: u64, + pub evaluation_interval_ms: u64, + pub raw_data_retained: bool, +} + +/// One `asap_run bench` run. +#[derive(Debug, Clone)] +pub struct BenchRequest<'a> { + pub workload: &'a PlanningWorkload, + pub arms: Vec, + pub settings: Vec, + pub series_count: u64, + pub duration_ms: u64, + /// Runs per arm; each measurement is their median. + pub repetitions: u32, + pub tables: PlanningTables, + pub seed: u64, + /// Whether the gate checks cost (`--no-cost-gate` turns it off; error + /// is always checked). + pub cost_gate: bool, + /// Where `runs///` goes; `None` writes nothing. + pub output_root: Option, +} + +/// What a run measured of one arm, per second of wall time unless the +/// name says otherwise. +#[derive(Debug, Clone, PartialEq)] +pub struct MeasuredUsage { + pub p50_query_ms: f64, + pub p95_query_ms: f64, + pub ingestion_cpu_seconds_per_second: f64, + pub query_cpu_seconds_per_second: f64, + pub average_memory_bytes: f64, + pub peak_memory_bytes: u64, + pub scanned_bytes_per_second: f64, +} + +/// One arm of one setting. +#[derive(Debug, Clone)] +pub struct ArmMeasurement { + pub arm: BenchArm, + pub setting: DataSetting, + pub plan: InstallablePlan, + /// The planner's prediction for this plan, with the same cost inputs. + pub predicted: SelectedPlanReport, + /// `Err` when the arm did not run. + pub measured: Result, + /// Cost model 1 over the measured usage at the planner's own prices: + /// the median of `repetition_costs_per_second`. + pub measured_cost_per_second: Option, + /// The same, per repetition, for the noise band. + pub repetition_costs_per_second: Vec, + /// The arm's error against the exact arm in the target's metric, at + /// the target's failure probability. + pub measured_error: Option, +} + +/// One gate row: one workload at one setting. +#[derive(Debug, Clone, PartialEq)] +pub struct GateRow { + pub workload: String, + pub setting: DataSetting, + /// `None` when the exact arm did not run. + pub exact_cost_per_second: Option, + /// `Err` when the selected arm did not run. + pub selected_cost_per_second: Result, + pub measured_error: f64, + pub target_error: f64, +} + +/// Why a gate failed. +#[derive(Debug, Clone, PartialEq)] +pub enum GateFailure { + /// The selected plan cost more than 1.05 × the exact plan. + CostAboveLimit { + workload: String, + setting: DataSetting, + }, + /// Its error exceeded the target. + ErrorAboveTarget { + workload: String, + setting: DataSetting, + }, + /// Its error is not a number. + ErrorNotANumber { + workload: String, + setting: DataSetting, + }, + ExactArmMissing { + workload: String, + setting: DataSetting, + }, + SelectedArmFailed { + workload: String, + setting: DataSetting, + }, + /// An expected row is absent. + MissingRow { + workload: String, + setting: DataSetting, + }, + /// A row nobody expected. + ExtraRow { + workload: String, + setting: DataSetting, + }, +} + +/// The gate's verdict over a run's rows. +#[derive(Debug, Clone, PartialEq)] +pub enum GateVerdict { + Pass, + Fail(Vec), +} + +/// The gate over `rows`, which must be exactly the `expected` (workload, +/// setting) pairs; `cost_gate` false checks error and completeness only. +pub fn gate_verdict( + expected: &[(String, DataSetting)], + rows: &[GateRow], + cost_gate: bool, +) -> GateVerdict { + let _ = (expected, rows, cost_gate); + todo!("asap_run: implement after test approval") +} + +/// What one benchmark run produced. +#[derive(Debug, Clone)] +pub struct BenchReport { + pub arms: Vec, + pub rows: Vec, + pub verdict: GateVerdict, + /// `runs///`, when written. + pub run_dir: Option, +} + +/// Run `request`. +pub fn bench(request: &BenchRequest<'_>) -> Result { + let _ = request; + todo!("asap_run: implement after test approval") +} + +/// The fixed showcase suite (`--suite showcase`), at the per-PR scale +/// when `small` (100 series, 10 minutes of data). +pub fn showcase_suite(small: bool) -> Vec<(String, PlanningWorkload, Vec)> { + let _ = small; + todo!("asap_run: implement after test approval") +} + +/// Run the showcase suite with `tables`. +pub fn bench_showcase( + small: bool, + tables: &PlanningTables, + output_root: Option, +) -> Result { + let _ = (small, tables, output_root); + todo!("asap_run: implement after test approval") +} + +/// Measure one given plan of `workload` at `setting` (the exhaustive +/// oracle runs every enumerated plan through it). +pub fn bench_plan( + plan: &InstallablePlan, + workload: &PlanningWorkload, + setting: &DataSetting, + request: &BenchRequest<'_>, +) -> Result { + let _ = (plan, workload, setting, request); + todo!("asap_run: implement after test approval") +} + +/// The installable plan of one plan over Stage 2's candidates: the +/// `candidates` chosen and the `edges` feeding each of their inputs (the +/// exhaustive oracle builds every enumerated plan with it). +pub fn installable_plan_of_selection( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + candidates: &BTreeSet, + edges: &BTreeSet, +) -> Result { + let _ = (physical, entries, candidates, edges); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/calibration_tables.rs b/crates/devtools/src/measurement/calibration_tables.rs new file mode 100644 index 000000000..07cdcc9c1 --- /dev/null +++ b/crates/devtools/src/measurement/calibration_tables.rs @@ -0,0 +1,130 @@ +//! A machine's calibration tables: running a calibration and writing its +//! directory, and finding and loading the tables a planning run reads +//! (shared by `asap_run` and `asap-planner`). +//! +//! Layout, under `ASAP_CALIBRATION_DIR` or `/calibration`: +//! `/{machine.json, costs.json, accuracy.json, +//! calibrate-run.json}`. Example: with `ASAP_CALIBRATION_DIR=/data/cal` on +//! machine `xeon-gold-6142-56c-251g`, the cost table is +//! `/data/cal/xeon-gold-6142-56c-251g/costs.json`, unless `--costs ` +//! names another. + +use std::path::{Path, PathBuf}; + +use asap_accuracy::MeasuredSketchAccuracy; +use asap_cost::{CostInputs, MeasuredOperationCosts}; + +use crate::measurement::accuracy_calibration::MergeCountErrors; +use crate::measurement::cost_calibration::{CalibrationFamilies, CostFitRecord}; +use crate::measurement::machine::MachineDescription; + +/// The environment variable naming the calibration directory. +pub const CALIBRATION_DIR_ENV: &str = "ASAP_CALIBRATION_DIR"; + +/// One `asap_run calibrate` run. +#[derive(Debug, Clone, PartialEq)] +pub struct CalibrateRequest { + pub families: CalibrationFamilies, + /// The per-PR scale: fewer sizes and trials. + pub small: bool, + pub seed: u64, +} + +/// What one calibration measured. +#[derive(Debug, Clone)] +pub struct Calibration { + pub machine: MachineDescription, + pub costs: MeasuredOperationCosts, + pub accuracy: MeasuredSketchAccuracy, + /// `calibrate-run.json`: every case's samples and fit. + pub cost_fits: Vec, + /// `calibrate-run.json`: the errors per merge count. + pub accuracy_by_merge_count: Vec, +} + +/// Run a calibration on this machine. +pub fn calibrate(request: &CalibrateRequest) -> Result { + let _ = request; + todo!("asap_run: implement after test approval") +} + +/// Write `calibration` to `//`, replacing an earlier +/// run's files; returns that directory. +pub fn write_calibration(dir: &Path, calibration: &Calibration) -> Result { + let _ = (dir, calibration); + todo!("asap_run: implement after test approval") +} + +/// The tables a run was told to read on its command line. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TableOverrides { + /// `--costs `. + pub costs: Option, + /// `--accuracy `. + pub accuracy: Option, + /// `--cost-inputs `: cost inputs, possibly with a cost table + /// inside. + pub cost_inputs: Option, +} + +/// Where one table is read from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TablePath { + /// Named on the command line. + Given(PathBuf), + /// The machine's file in the calibration directory. + Calibrated(PathBuf), + /// No file: the run falls back to the formulas, warning that this + /// path was looked at. + Missing { looked_at: PathBuf }, +} + +/// Where a run's two tables are read from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedTablePaths { + pub costs: TablePath, + pub accuracy: TablePath, +} + +/// Where the tables of machine `machine_key` are read from: +/// `overrides` first, then the calibration directory (`env_dir`, from +/// [`CALIBRATION_DIR_ENV`], else `/calibration`). +pub fn resolve_table_paths( + overrides: &TableOverrides, + env_dir: Option<&Path>, + repo_root: Option<&Path>, + machine_key: &str, +) -> ResolvedTablePaths { + let _ = (overrides, env_dir, repo_root, machine_key); + todo!("asap_run: implement after test approval") +} + +/// The tables a planning run plans with. +#[derive(Debug, Clone)] +pub struct PlanningTables { + pub cost_inputs: CostInputs, + pub accuracy: MeasuredSketchAccuracy, + /// What the run prints to standard error, e.g. that no table exists + /// for this machine. + pub warnings: Vec, +} + +/// Why a table could not be read: a corrupt or invalid file. +#[derive(Debug, Clone, PartialEq)] +pub struct TableLoadError { + pub path: PathBuf, + pub reason: String, +} + +/// Load the tables `resolved` names, over the cost inputs +/// `overrides.cost_inputs` names (the defaults without one); a missing +/// table falls back to the formulas with a warning naming `machine_key` +/// and `asap_run calibrate`. +pub fn load_planning_tables( + resolved: &ResolvedTablePaths, + overrides: &TableOverrides, + machine_key: &str, +) -> Result { + let _ = (resolved, overrides, machine_key); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/cost_calibration.rs b/crates/devtools/src/measurement/cost_calibration.rs new file mode 100644 index 000000000..705507143 --- /dev/null +++ b/crates/devtools/src/measurement/cost_calibration.rs @@ -0,0 +1,165 @@ +//! Cost calibration (`asap_run calibrate`): microbenchmark every operator +//! kind × phase × state family on the executor, at several sizes, and fit +//! the planner's per-node model `cpu_ns = units · ns_per_unit + +//! ns_per_invocation` to the measurements. +//! +//! Example: an operator measured at 1,000, 10,000 and 100,000 units taking +//! 60,000, 510,000 and 5,010,000 ns fits `ns_per_unit = 50` and +//! `ns_per_invocation = 10,000`. + +use asap_cost::{MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_planner::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::summary_type::{ExactKind, SketchParams}; +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; + +/// One measurement: the units one invocation processed and the thread CPU +/// time it took. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct CostSample { + pub units: f64, + pub cpu_ns: f64, +} + +/// The fitted per-node model and what the fit noticed. +#[derive(Debug, Clone, PartialEq)] +pub struct FittedOperatorCost { + pub ns_per_unit: f64, + pub ns_per_invocation: f64, + /// Empty for a clean fit. + pub flags: Vec, +} + +/// Something about a fit `calibrate-run.json` records. +#[derive(Debug, Clone, PartialEq)] +pub enum FitFlag { + /// The unconstrained intercept was negative (`raw_intercept_ns`); it + /// was set to 0 and the slope refitted through the origin. + NegativeInterceptClamped { raw_intercept_ns: f64 }, + /// The fit misses the measurement at `units` by `relative_residual`, + /// more than the tolerance. + ResidualAboveTolerance { units: f64, relative_residual: f64 }, + /// Fewer distinct sizes, or a narrower span of sizes, than a fit needs. + TooFewSizes { sizes: usize }, +} + +/// Fit `cpu_ns = units · ns_per_unit + ns_per_invocation` to `samples`. +pub fn fit_operator_cost(samples: &[CostSample]) -> FittedOperatorCost { + let _ = samples; + todo!("asap_run: implement after test approval") +} + +/// A per-thread CPU clock. +pub trait CpuClock { + /// CPU nanoseconds this thread has used. + fn thread_cpu_ns(&self) -> u64; +} + +/// The operating system's per-thread CPU clock. +#[derive(Debug, Clone, Copy, Default)] +pub struct ThreadCpuClock; + +impl CpuClock for ThreadCpuClock { + fn thread_cpu_ns(&self) -> u64 { + todo!("asap_run: implement after test approval") + } +} + +/// One operator invocation a calibration times, at a given size. +pub trait CalibrationKernel { + /// Run one invocation processing `units` units, counted as the cost + /// model counts them (`OperatorKind`). + fn run(&mut self, units: u64); +} + +/// Time `kernel` on `clock` at each of `sizes`, `repetitions` times each, +/// one [`CostSample`] per size (the median of its repetitions). +pub fn measure_kernel( + kernel: &mut dyn CalibrationKernel, + clock: &dyn CpuClock, + sizes: &[u64], + repetitions: u32, +) -> Vec { + let _ = (kernel, clock, sizes, repetitions); + todo!("asap_run: implement after test approval") +} + +/// The state family an entry is for. +#[derive(Debug, Clone, PartialEq)] +pub enum CalibratedFamily { + None, + Sketch(SketchParams), + Exact(ExactKind), + /// An `Aggregate` measure, by its JSON `kind`. + Measure(String), +} + +/// One (operator, phase, family) a calibration measures. +#[derive(Debug, Clone, PartialEq)] +pub struct CalibrationCase { + pub operator: OperatorKind, + pub phase: ExecutionTiming, + pub family: CalibratedFamily, +} + +/// Which families a calibration measures. +#[derive(Debug, Clone, PartialEq)] +pub enum CalibrationFamilies { + All, + /// The exact accumulators and aggregate measures only. + Exact, + Only(Vec), +} + +/// The physical DAG a case's microbenchmark runs at one size. +#[derive(Debug, Clone)] +pub struct CalibrationDag { + pub dag: PhysicalASAPDAG, + pub entries: Vec, + pub data: DataWorkload, + /// The node the case times. + pub measured_node: PhysicalASAPNodeId, + /// The units that node processes per invocation, as the + /// microbenchmark counts them. + pub units: u64, +} + +/// Every case of `families`: each `OperatorKind` × phase × family. +pub fn calibration_cases(families: &CalibrationFamilies) -> Vec { + let _ = families; + todo!("asap_run: implement after test approval") +} + +impl CalibrationCase { + /// The DAG this case's microbenchmark runs at about `units` units. + pub fn dag_at(&self, units: u64) -> CalibrationDag { + let _ = units; + todo!("asap_run: implement after test approval") + } + + /// The table entry of this case with `fit`. + pub fn operator_cost( + &self, + fit: &FittedOperatorCost, + memory_bytes_per_state: Option, + ) -> OperatorCost { + let _ = (fit, memory_bytes_per_state); + todo!("asap_run: implement after test approval") + } +} + +/// One case's measurements and fit, as `calibrate-run.json` records them. +#[derive(Debug, Clone, PartialEq)] +pub struct CostFitRecord { + pub case: CalibrationCase, + pub samples: Vec, + pub fit: FittedOperatorCost, + pub memory_bytes_per_state: Option, +} + +/// The cost table of `records`, with the per-request overhead +/// `ns_per_evaluation`. +pub fn cost_table_of(records: &[CostFitRecord], ns_per_evaluation: f64) -> MeasuredOperationCosts { + let _ = (records, ns_per_evaluation); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/execution.rs b/crates/devtools/src/measurement/execution.rs new file mode 100644 index 000000000..8017111d5 --- /dev/null +++ b/crates/devtools/src/measurement/execution.rs @@ -0,0 +1,140 @@ +//! The reference driver of the executor (`asap_run execute`): plan a +//! workload, install the selected plan, ingest every refresh window in time +//! order (the executor's `execute_ingestion`, one ingestion-scope run per +//! window) and evaluate every query root at the requested times (its +//! `execute`, one query-scope run per evaluation time, whose root streams +//! are drained in the request's `RootDrainOrder`). +//! +//! Example: `sum by (job) (sum_over_time(m[1m]))` over four 1 Hz series of +//! constant values 1, 2 (`job="a"`) and 3, 4 (`job="b"`), evaluated at +//! t = 600 s, answers `{job="a"}: 180` and `{job="b"}: 420`: the window +//! `(540 s, 600 s]` holds 60 samples of each series. + +use std::collections::BTreeMap; + +use asap_accuracy::MeasuredSketchAccuracy; +use asap_cost::CostInputs; +use asap_planner::{ + DeploymentCapabilities, InstallablePlan, MaterializedViewRefresh, PhysicalASAPNodeId, + PlanningWorkload, SelectedPlanReport, +}; + +use crate::measurement::samples::{GeneratedDataSettings, Sample, SeriesLabels}; + +/// Where a run's samples come from. +#[derive(Debug, Clone, PartialEq)] +pub enum DataInput { + /// Generated by `aqpbm-datagen` from each source's `SourceFacts`. + GeneratedFromFacts(GeneratedDataSettings), + /// Given, e.g. read from a JSONL file. + Samples(Vec), +} + +/// In which order a query evaluation drains its roots' streams. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RootDrainOrder { + /// Root 0 first, then root 1, and so on. + WorkloadOrder, + /// The last root first. + Reversed, + /// Every root at once. + Concurrent, +} + +/// One `asap_run execute` run. +#[derive(Debug, Clone)] +pub struct ExecuteRequest<'a> { + pub workload: &'a PlanningWorkload, + pub data: DataInput, + /// Each evaluation time, in ms, ascending; ingestion starts at the + /// data's first sample. + pub evaluation_times_ms: Vec, + pub deployment_capabilities: DeploymentCapabilities, + pub cost_inputs: CostInputs, + pub accuracy: MeasuredSketchAccuracy, + pub root_drain_order: RootDrainOrder, +} + +impl<'a> ExecuteRequest<'a> { + /// A run of `workload` over `data` at `evaluation_times_ms`, with + /// raw data retained, the default cost inputs, no measured accuracy + /// and roots drained in workload order. + pub fn new( + workload: &'a PlanningWorkload, + data: DataInput, + evaluation_times_ms: Vec, + ) -> Self { + Self { + workload, + data, + evaluation_times_ms, + deployment_capabilities: DeploymentCapabilities::default(), + cost_inputs: CostInputs::default(), + accuracy: MeasuredSketchAccuracy::default(), + root_drain_order: RootDrainOrder::WorkloadOrder, + } + } +} + +/// What one run did and answered. +#[derive(Debug, Clone)] +pub struct ExecutionReport { + /// The plan installed. + pub plan: InstallablePlan, + /// Its predicted cost. + pub predicted: SelectedPlanReport, + /// One per evaluation time, in request order. + pub evaluations: Vec, + /// One per materialized view of the plan. + pub views: Vec, +} + +/// The answers of one evaluation. +#[derive(Debug, Clone, PartialEq)] +pub struct EvaluationAnswers { + pub at_ms: i64, + /// One per query root, in workload order. + pub per_root: Vec, +} + +/// One root's answer: a value per output series. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct RootAnswers { + pub values: BTreeMap, +} + +/// What one materialized view did over the run. +#[derive(Debug, Clone, PartialEq)] +pub struct ViewActivity { + pub node_id: PhysicalASAPNodeId, + pub refresh: MaterializedViewRefresh, + /// How many times it was refreshed. + pub refresh_count: u64, + /// The states it held after each evaluation, in evaluation order. + pub states_after_evaluation: Vec, + /// The bytes its states held after each evaluation (each kernel's + /// `memory_bytes()`). + pub bytes_after_evaluation: Vec, +} + +/// Why a run did not answer. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum ExecuteError { + /// Planning or installation failed. + Plan(String), + /// The data: unreadable, or not generable from the facts. + Data(String), + /// An evaluation at `at_ms` before the data covers its warm-up (the + /// longest retention or raw lookback plus the refresh width), which + /// starts at `first_covered_ms`. + BeforeWarmUp { at_ms: i64, first_covered_ms: i64 }, + /// The executor failed. + Executor(String), +} + +/// Run `request`: plan, install, ingest and evaluate. +pub fn execute(request: &ExecuteRequest<'_>) -> Result { + let _ = request; + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/machine.rs b/crates/devtools/src/measurement/machine.rs new file mode 100644 index 000000000..2bb4af966 --- /dev/null +++ b/crates/devtools/src/measurement/machine.rs @@ -0,0 +1,36 @@ +//! The machine a calibration is measured on, and the key its tables are +//! stored under: CPU model, logical cores and memory, never the hostname. +//! +//! Example: an "Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz" with 56 logical +//! cores and 263,724,000 KiB of memory is `xeon-gold-6142-56c-251g`. + +use serde::{Deserialize, Serialize}; + +/// What `machine.json` records. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MachineDescription { + pub cpu_model: String, + pub logical_cores: u32, + /// `MemTotal`, in KiB. + pub memory_kib: u64, + pub os: String, +} + +impl MachineDescription { + /// The machine `/proc/cpuinfo` and `/proc/meminfo` describe. + pub fn from_proc(cpuinfo: &str, meminfo: &str, os: &str) -> Result { + let _ = (cpuinfo, meminfo, os); + todo!("asap_run: implement after test approval") + } + + /// This machine. + pub fn detect() -> Result { + todo!("asap_run: implement after test approval") + } + + /// The directory name its tables are stored under: `[a-z0-9-]+`. + pub fn machine_key(&self) -> String { + todo!("asap_run: implement after test approval") + } +} diff --git a/crates/devtools/src/measurement/mod.rs b/crates/devtools/src/measurement/mod.rs new file mode 100644 index 000000000..98eb1c6fe --- /dev/null +++ b/crates/devtools/src/measurement/mod.rs @@ -0,0 +1,33 @@ +//! Running selected plans on the Planner's executor and measuring them, +//! behind the `asap_run` binary: +//! +//! - [`execution`]: the reference driver. Plan a workload, install the plan +//! (its materialized views and ingestion refresh groups), ingest each +//! refresh window, evaluate at given times and read every root's answers +//! (`asap_run execute`). +//! - [`samples`]: the data a run ingests, read from a JSONL file or +//! generated from the workload's source facts with `aqpbm-datagen`. +//! - [`cost_calibration`] and [`accuracy_calibration`]: microbenchmarks of +//! every operator, phase and state family, fitted into the planner's +//! `MeasuredOperationCosts`, and seeded trials of every sketch +//! configuration, recorded as `MeasuredSketchAccuracy` (`asap_run +//! calibrate`). +//! - [`machine`] and [`calibration_tables`]: the machine key a calibration +//! is stored under, and which tables a planning run reads. +//! - [`benchmark`]: each arm (exact, planned, no raw data) end to end, the +//! planner's predicted cost beside the measured one, and the gate +//! (`asap_run bench`). +//! - [`source_facts`]: `SourceFacts` computed from data, like a database's +//! `ANALYZE` (`asap_run facts`). +//! +//! The planner library only reads `CostInputs`, an `AccuracyEvidence` and +//! `SourceFacts`; every path, machine and data adapter lives here. + +pub mod accuracy_calibration; +pub mod benchmark; +pub mod calibration_tables; +pub mod cost_calibration; +pub mod execution; +pub mod machine; +pub mod samples; +pub mod source_facts; diff --git a/crates/devtools/src/measurement/samples.rs b/crates/devtools/src/measurement/samples.rs new file mode 100644 index 000000000..2e8da766e --- /dev/null +++ b/crates/devtools/src/measurement/samples.rs @@ -0,0 +1,79 @@ +//! The samples a run ingests: read from a JSONL file, one sample per line, +//! or generated with `aqpbm-datagen` from the workload's source facts. +//! +//! Example line: `{"labels": {"__name__": "m", "job": "a"}, "ts_ms": 1000, +//! "value": 2.0}`: series `m{job="a"}` has value 2 at t = 1 s. + +use std::collections::BTreeMap; + +use aqpbm_datagen::{GeneratedTimeSeries, TimeSeriesDescription}; +use asap_types::workload::{DataWorkload, SourceFacts}; +use serde::{Deserialize, Serialize}; + +/// A series' labels, the metric name under `__name__`. +pub type SeriesLabels = BTreeMap; + +/// The label holding a PromQL series' metric name. +pub const METRIC_NAME_LABEL: &str = "__name__"; + +/// One sample of one series. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Sample { + pub labels: SeriesLabels, + pub ts_ms: i64, + pub value: f64, +} + +/// Why a JSONL file is not a list of samples. +#[derive(Debug, Clone, PartialEq)] +pub struct SampleReadError { + /// 1-based line number. + pub line: usize, + pub reason: String, +} + +/// The samples of a JSONL text, in file order. +pub fn read_samples_jsonl(text: &str) -> Result, SampleReadError> { + let _ = text; + todo!("asap_run: implement after test approval") +} + +/// How `aqpbm-datagen` generates data from source facts. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct GeneratedDataSettings { + pub seed: u64, + pub start_ms: i64, + pub duration_ms: u64, +} + +/// The description `aqpbm-datagen` generates source `metric` from: +/// `series_count` series, one sample per `sample_interval`, each label's +/// `distinct_values_per_label` as its cardinality. +pub fn time_series_description_of( + metric: &str, + facts: &SourceFacts, + settings: &GeneratedDataSettings, +) -> Result { + let _ = (metric, facts, settings); + todo!("asap_run: implement after test approval") +} + +/// The samples of every source `data` declares, generated in time order. +pub fn generate_samples( + data: &DataWorkload, + settings: &GeneratedDataSettings, +) -> Result, String> { + let _ = (data, settings); + todo!("asap_run: implement after test approval") +} + +/// The samples of `generated`, as series of metric `metric`. An error when +/// the generator yields one, e.g. a counter whose running sum passed 2^53. +pub fn samples_of_generated( + metric: &str, + generated: &GeneratedTimeSeries, +) -> Result, String> { + let _ = (metric, generated); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/src/measurement/source_facts.rs b/crates/devtools/src/measurement/source_facts.rs new file mode 100644 index 000000000..090f05213 --- /dev/null +++ b/crates/devtools/src/measurement/source_facts.rs @@ -0,0 +1,108 @@ +//! Source facts from data (`asap_run facts`), like a database's `ANALYZE`: +//! per-source statistics over the samples, written into the workload's +//! `data_workload.sources`. +//! +//! Facts that affect only cost are written as observed. Facts a guarantee +//! relies on are printed as suggestions with a ×2 margin, and written only +//! when the caller trusts observed bounds: the next hour's data may exceed +//! an observed value, and then the guarantee is false. +//! +//! Example: four series `m{job, instance}` with samples 1 s apart give +//! `series_count: 4` and `sample_interval: 1000` (cost-only, written), and, +//! when a query has a whole-result target, a suggested +//! `distinct_values_per_label: {"job": 4}` for 2 observed job values. + +use std::collections::BTreeMap; + +use asap_types::stages::ValueSign; +use asap_types::workload::{DataWorkload, LabelSetDistinctValues, PlanningWorkload, StreamNorms}; + +use crate::measurement::samples::Sample; + +/// The smallest and largest non-zero absolute value of a column. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ObservedMagnitude { + pub min_abs: f64, + pub max_abs: f64, +} + +/// What one source's samples show. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ObservedSourceStatistics { + /// Distinct label sets. + pub series_count: u64, + /// The median of the per-series median gaps; `None` when no series has + /// two samples. + pub sample_interval_ms: Option, + /// Distinct values per label, a missing label counting as one more. + pub distinct_values_per_label: BTreeMap, + /// Distinct combinations of each `by (…)` set of two or more labels the + /// workload's queries use. + pub distinct_values_per_label_set: Vec, + /// Per column; a column of zeros only has none. + pub column_magnitudes: BTreeMap, + /// Per column, by sign bit; `Unknown` for mixed signs. + pub column_signs: BTreeMap, + /// The most distinct values one evaluation window of the workload's + /// longest window saw, per output group. + pub max_distinct_per_evaluation: Option, + /// The largest norms of one group's stream over the longest window. + pub stream_norms: Option, +} + +/// Each source's statistics over `samples`, keyed by metric name. +pub fn observe_source_statistics( + samples: &[Sample], + workload: &PlanningWorkload, +) -> BTreeMap { + let _ = (samples, workload); + todo!("asap_run: implement after test approval") +} + +/// One guarantee fact, suggested with its margin. +#[derive(Debug, Clone, PartialEq)] +pub enum SuggestedFact { + ColumnSign { column: String, sign: ValueSign }, + SeriesCount(u64), + DistinctValuesPerLabel { label: String, distinct_values: u64 }, + DistinctValuesPerLabelSet(LabelSetDistinctValues), + MaxDistinctPerEvaluation(u64), + StreamNorms(StreamNorms), +} + +/// A suggestion for one source. +#[derive(Debug, Clone, PartialEq)] +pub struct GuaranteeFactSuggestion { + pub source: String, + pub fact: SuggestedFact, +} + +/// How `asap_run facts` writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct FactsOptions { + /// `--trust-observed-bounds`: write the suggestions too. + pub trust_observed_bounds: bool, +} + +/// What `asap_run facts` produced. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceFactsReport { + pub observed: BTreeMap, + /// The workload's data workload with the facts written. + pub data_workload: DataWorkload, + /// The magnitudes written per source and column, with their margin; + /// they go into `SourceFacts::column_magnitudes` once it exists. + pub column_magnitudes: BTreeMap>, + /// The guarantee facts, printed; written too when trusted. + pub suggestions: Vec, +} + +/// The facts of `samples` for `workload`. +pub fn source_facts_of_samples( + samples: &[Sample], + workload: &PlanningWorkload, + options: FactsOptions, +) -> Result { + let _ = (samples, workload, options); + todo!("asap_run: implement after test approval") +} diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index ed12546d0..13ca11416 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -5,5 +5,6 @@ mod asap_planner; mod asap_planner_acceptance; mod cross_language; +mod measurement; mod planner_fuzz; mod viewer_contract; diff --git a/crates/devtools/tests/measurement/common.rs b/crates/devtools/tests/measurement/common.rs new file mode 100644 index 000000000..bd9c72cf0 --- /dev/null +++ b/crates/devtools/tests/measurement/common.rs @@ -0,0 +1,200 @@ +//! Fixtures shared by the `asap_run` tests: workloads, hand-made samples +//! and planning through the facade. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use asap_accuracy::AccuracyEvidence; +use asap_cost::CostInputs; +use asap_devtools::measurement::samples::{Sample, METRIC_NAME_LABEL}; +use asap_planner::{ + DeploymentCapabilities, InstallablePlan, PlanningModels, PlanningWorkload, SelectedPlanReport, +}; +use serde_json::{json, Value}; + +/// An exact accuracy requirement. +pub fn exact() -> Value { + json!({"max_error": 0.0, "failure_probability": 0.0}) +} + +/// Within `max_error` in `metric`, per value, failing with probability at +/// most `failure_probability`. +pub fn per_value(metric: &str, max_error: f64, failure_probability: f64) -> Value { + json!({ + "accuracy_metric": metric, + "max_error": max_error, + "failure_probability": failure_probability, + "accuracy_scope": "per_value", + }) +} + +/// Within `max_error` in `metric` for the whole result. +pub fn whole_result(metric: &str, max_error: f64, failure_probability: f64) -> Value { + json!({ + "accuracy_metric": metric, + "max_error": max_error, + "failure_probability": failure_probability, + "accuracy_scope": "whole_result", + }) +} + +/// The facts of a source of `series` series sampled every +/// `sample_interval_ms`. +pub fn facts(series: u64, sample_interval_ms: u64) -> Value { + json!({"series_count": series, "sample_interval": sample_interval_ms}) +} + +/// Repeating PromQL `queries` (each with its accuracy), every +/// `every_ms`, over `sources` (metric name → facts). The data arrives at +/// the rate the facts declare (each source's `series_count` per +/// `sample_interval`), unknown when none declares one. +pub fn workload( + queries: &[(&str, Value)], + every_ms: u64, + sources: &[(&str, Value)], +) -> PlanningWorkload { + let rows_per_second: f64 = sources + .iter() + .filter_map(|(_, facts)| { + Some(facts["series_count"].as_f64()? * 1_000.0 / facts["sample_interval"].as_f64()?) + }) + .sum(); + let arrival = match rows_per_second > 0.0 { + true => json!({"ingesting": {"rows_per_second": rows_per_second}}), + false => json!("unknown"), + }; + let value = json!({ + "query_workload": {"language": "promql", "query_batch": null, "repeating_queries": + queries.iter().map(|(query, accuracy)| json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": every_ms, "evaluation_phase": 0}}, + "requirements": {"accuracy": accuracy}, + "predictability": {"predictable": {"known_at": null}}, + })).collect::>()}, + "data_workload": { + "arrival": arrival, + "sources": sources.iter().map(|(name, facts)| (name.to_string(), facts.clone())) + .collect::>(), + }, + }); + serde_json::from_value(value).unwrap() +} + +/// One sample of metric `metric` with `labels`. +pub fn sample(metric: &str, labels: &[(&str, &str)], ts_ms: i64, value: f64) -> Sample { + let mut all: BTreeMap = labels + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + all.insert(METRIC_NAME_LABEL.into(), metric.into()); + Sample { + labels: all, + ts_ms, + value, + } +} + +/// The `job` label set of an answer. +pub fn job(name: &str) -> BTreeMap { + [("job".to_string(), name.to_string())].into() +} + +/// Four 1 Hz series of `m` from t = 0 to `until_ms` inclusive, of constant +/// values 1 and 2 (`job="a"`, instances 1 and 2) and 3 and 4 (`job="b"`), +/// with `value_of(series, ts_ms)` overriding a value (series 0 to 3). +pub fn four_constant_series( + until_ms: i64, + value_of: impl Fn(usize, i64) -> Option, +) -> Vec { + let series = [ + ("a", "1", 1.0), + ("a", "2", 2.0), + ("b", "3", 3.0), + ("b", "4", 4.0), + ]; + let mut samples = Vec::new(); + for ts_ms in (0..=until_ms).step_by(1_000) { + for (index, (job, instance, value)) in series.iter().enumerate() { + samples.push(sample( + "m", + &[("job", job), ("instance", instance)], + ts_ms, + value_of(index, ts_ms).unwrap_or(*value), + )); + } + } + samples +} + +/// The facts of [`four_constant_series`]. +pub fn four_series_facts() -> Value { + json!({ + "series_count": 4, + "sample_interval": 1000, + "distinct_values_per_label": {"job": 2, "instance": 4}, + }) +} + +/// Plan `workload` as a deployment runtime does. +pub fn plan( + workload: &PlanningWorkload, + cost_inputs: &CostInputs, + evidence: &dyn AccuracyEvidence, + capabilities: &DeploymentCapabilities, +) -> (InstallablePlan, SelectedPlanReport) { + let models = PlanningModels::new(evidence) + .with_cost_inputs(cost_inputs) + .with_deployment_capabilities(capabilities); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(asap_planner::plan_installable( + asap_planner::UserInput::new(workload, asap_planner::FrontendInput::Promql, models), + )) + .unwrap() +} + +/// Raw data not retained: every window is kept in ingestion-time state. +pub fn no_raw_data() -> DeploymentCapabilities { + DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + } +} + +/// A fresh empty directory under the system temp directory, removed when +/// dropped. +pub struct TempDir(pub PathBuf); + +impl TempDir { + pub fn new(name: &str) -> Self { + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "asap_run_test_{name}_{}_{call}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } + + pub fn write(&self, relative: &str, contents: &str) -> PathBuf { + let path = self.0.join(relative); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, contents).unwrap(); + path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Whether `a` and `b` agree to the exact-arm tolerance. +// provisional answer 21: exact plans agree within 1e-9 relative. +pub fn exact_match(a: f64, b: f64) -> bool { + (a - b).abs() <= 1e-9 * a.abs().max(b.abs()).max(1.0) +} diff --git a/crates/devtools/tests/measurement/cost_accounting.rs b/crates/devtools/tests/measurement/cost_accounting.rs new file mode 100644 index 000000000..d6f121da1 --- /dev/null +++ b/crates/devtools/tests/measurement/cost_accounting.rs @@ -0,0 +1,431 @@ +//! §2.3 and §2.8: the planner prices from the measured table, state +//! bytes, and a shared node charged once. + +use asap_accuracy::NoMeasuredGuarantees; +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::measurement::execution::{execute, DataInput, ExecuteRequest}; +use asap_executor::factory::create_planner_accumulator; +use asap_planner::{DeploymentCapabilities, MaterializedViewRefresh, PlanningWorkload}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchParams, SummaryInputExpr, SummaryUpdate, +}; + +use super::common::{ + exact, facts, four_constant_series, four_series_facts, no_raw_data, per_value, plan, workload, +}; + +/// DDSketch builds (`SummaryAgg` and the tumbling-window chain) at +/// `ns_per_unit` per row, both phases. +fn ddsketch_build_table(ns_per_unit: f64) -> MeasuredOperationCosts { + let entry = |operator| OperatorCost { + operator, + phase: None, + sketch: Some(SketchParams::DDSketch { alpha: 0.01 }), + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }; + MeasuredOperationCosts { + operators: vec![ + entry(OperatorKind::SummaryAgg), + entry(OperatorKind::SummaryTumblingWindowChain), + ], + ..MeasuredOperationCosts::default() + } +} + +/// `quantile_over_time(0.99, latency[1h])` every 10 s over 1,000 series +/// sampled each second, within 1% per value at δ = 0.01. +fn quantile_workload() -> PlanningWorkload { + workload( + &[( + "quantile_over_time(0.99, latency[1h])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("latency", facts(1_000, 1_000))], + ) +} + +/// The same plan priced with table A (DDSketch builds at 1 ns per row) and +/// with A's builds at 10 ns: each build node's CPU rises by its rows × 9 ns. +/// A query-time build reads 1,000 series × 3,600 samples per evaluation, +/// 0.1 evaluations per second: Δ = 3.6e6 · 0.1 · 9 ns = 3.24e-3 core-s/s. +/// An ingestion-time build reads the 1,000 arriving rows per second: +/// Δ = 1,000 · 9 ns = 9e-6 core-s/s. +#[test] +fn planner_prices_from_table() { + let workload = quantile_workload(); + let table_a = CostInputs { + measured_operation_costs: ddsketch_build_table(1.0), + ..CostInputs::default() + }; + let table_a10 = CostInputs { + measured_operation_costs: ddsketch_build_table(10.0), + ..CostInputs::default() + }; + let (installable, _) = plan( + &workload, + &table_a, + &NoMeasuredGuarantees, + &DeploymentCapabilities::default(), + ); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap(); + let (cost_a, _) = asap_cost::plan_cost(&installable.dag, &entries, &data, &table_a).unwrap(); + let (cost_a10, _) = + asap_cost::plan_cost(&installable.dag, &entries, &data, &table_a10).unwrap(); + let builds: Vec<_> = installable + .dag + .nodes + .iter() + .filter(|node| matches!(node.parameters, Some(SketchParams::DDSketch { .. }))) + .filter(|node| { + matches!( + node.payload, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } + ) + ) + }) + .collect(); + assert!( + !builds.is_empty(), + "the plan builds no DDSketch: {:?}", + installable.dag + ); + for node in builds { + let expected = match node.output_state.timing { + ExecutionTiming::QueryTime => 3.6e6 * 0.1 * 9e-9, + ExecutionTiming::IngestionTime => 1_000.0 * 9e-9, + }; + let delta = cost_a10.per_node[&node.id].cpu_core_seconds_per_second + - cost_a.per_node[&node.id].cpu_core_seconds_per_second; + assert!( + (delta - expected).abs() <= 1e-6 * expected, + "node {}: {delta} vs {expected}", + node.id + ); + } +} + +/// After a small calibration, every node of every small showcase plan is +/// priced from the table: no operator is left to the formulas. +#[test] +fn calibrated_table_covers_showcase_plans() { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: true, + seed: 7, + }) + .unwrap(); + let inputs = CostInputs { + measured_operation_costs: calibration.costs, + ..CostInputs::default() + }; + for (name, workload, settings) in asap_devtools::measurement::benchmark::showcase_suite(true) { + for setting in settings { + let caps = DeploymentCapabilities { + raw_data_retained: setting.raw_data_retained, + ..DeploymentCapabilities::default() + }; + let (_, report) = plan(&workload, &inputs, &calibration.accuracy, &caps); + for node in &report.predicted_usage.per_node { + assert_eq!( + node.cost_source, + asap_cost::NodeCostSource::Measured, + "{name} {setting:?}: {:?} at {:?}", + node.operator, + node.timing + ); + } + } + } +} + +/// `sum by (job) (sum_over_time(m[1h]))` every 10 s without raw data, +/// with a measured merge of 1 ns per state and 10 µs per invocation: a +/// 10 s chain merges 360 windows per evaluation, a 1 s chain 3,600, so the +/// planner keeps the 10 s chain. +/// +/// The plan's version of this test compares invocation costs at equal +/// units; the cost model charges an ingestion-time `ns_per_invocation` +/// per second of arriving rows, whatever the chain width, so here only +/// the merged states differ. +// provisional answer 10 +#[test] +fn invocation_cost_ranks_chain_widths() { + let merge = |phase| OperatorCost { + operator: OperatorKind::SummaryMerge, + phase: Some(phase), + sketch: None, + exact: Some(ExactKind::Sum), + measure: None, + ns_per_unit: 1.0, + ns_per_invocation: 10_000.0, + memory_bytes_per_state: None, + }; + let inputs = CostInputs { + measured_operation_costs: MeasuredOperationCosts { + operators: vec![ + merge(ExecutionTiming::QueryTime), + merge(ExecutionTiming::IngestionTime), + ], + ..MeasuredOperationCosts::default() + }, + ..CostInputs::default() + }; + let workload = workload( + &[("sum by (job) (sum_over_time(m[1h]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let (installable, _) = plan(&workload, &inputs, &NoMeasuredGuarantees, &no_raw_data()); + let widths: Vec = installable + .dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + _ => None, + }) + .collect(); + assert_eq!(widths, vec![10_000]); +} + +/// The bytes the executor charges one state of `family` with `parameters` +/// after one update: of the sample value, or (`keyed`) of one item. +fn kernel_state_bytes( + family: FieldDataType, + parameters: Option<&SketchParams>, + keyed: bool, +) -> u64 { + let value = SummaryUpdate::column(ColumnRef::SampleValue); + let input = match keyed { + false => value, + true => SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), + ..value + }, + }; + let mut updater = create_planner_accumulator(&family, parameters, &input).unwrap(); + match keyed { + false => updater.update_single(1.0, 0).unwrap(), + true => updater + .update_keyed(&["checkout".to_string()], 1.0, 0) + .unwrap(), + } + updater.into_accumulator().approx_memory_bytes() as u64 +} + +/// `sum by (job) (sum_over_time(m[1m]))` as a 10 s chain holds 6 windows × +/// 2 groups of one exact-sum state each: 12 · B_sum bytes, B_sum the +/// kernel's own charge for one state. The measured view bytes and the +/// planner's average memory are within 1.5× of it. +#[test] +fn memory_per_state_hand() { + let state_bytes = kernel_state_bytes( + FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + None, + false, + ); + let expected = 12.0 * state_bytes as f64; + let workload = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let mut request = ExecuteRequest::new( + &workload, + DataInput::Samples(four_constant_series(700_000, |_, _| None)), + vec![650_000, 700_000], + ); + request.deployment_capabilities = no_raw_data(); + let report = execute(&request).unwrap(); + let view = report + .views + .iter() + .find(|view| view.refresh == MaterializedViewRefresh::AtIngestion) + .expect("an ingestion-time view"); + let measured = *view.bytes_after_evaluation.last().unwrap() as f64; + assert!( + measured <= 1.5 * expected && expected <= 1.5 * measured, + "{measured} vs {expected}" + ); + let predicted = report.predicted.predicted_usage.average_memory_bytes; + assert!( + predicted <= 1.5 * expected && expected <= 1.5 * predicted, + "{predicted} vs {expected}" + ); +} + +/// Calibration writes, for a fixed-size state, the kernel's own charge: +/// Count-Min 272 × 5 holds what `create_planner_accumulator` charges +/// for it, at least d · w · 8 bytes of counters. +// provisional answer 9 +#[test] +fn fixed_size_state_bytes() { + let cms = SketchParams::Cms { + width: 272, + depth: 5, + }; + let charged = kernel_state_bytes( + FieldDataType::Sketch(SketchAlgorithm::Cms), + Some(&cms), + true, + ); + assert!(charged >= 272 * 5 * 8, "{charged}"); + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::Only(vec!["Cms".into()]), + small: true, + seed: 7, + }) + .unwrap(); + let entry = calibration + .costs + .operators + .iter() + .find(|entry| { + entry.operator == OperatorKind::SummaryAgg && entry.sketch.as_ref() == Some(&cms) + }) + .expect("an entry for Count-Min 272 x 5"); + assert_eq!(entry.memory_bytes_per_state, Some(charged as f64)); +} + +/// Two queries over one 10 s chain without raw data, +/// `sum by (job) (sum_over_time(m[5m]))` and the same over 10m: the joint +/// plan builds the chain once, and charges it once (one `per_node` entry, +/// read by both queries). +#[test] +fn shared_node_priced_once() { + let workload = workload( + &[ + ("sum by (job) (sum_over_time(m[5m]))", exact()), + ("sum by (job) (sum_over_time(m[10m]))", exact()), + ], + 10_000, + &[("m", four_series_facts())], + ); + let (installable, report) = plan( + &workload, + &CostInputs::default(), + &NoMeasuredGuarantees, + &no_raw_data(), + ); + let chains: Vec<_> = installable + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + }) + .collect(); + assert_eq!(chains.len(), 1, "{:?}", installable.dag); + let chain = chains[0].id; + let priced = report + .predicted_usage + .per_node + .iter() + .filter(|node| node.node_id == chain) + .count(); + assert_eq!(priced, 1); + // Both roots read the chain. + for &root in &installable.dag.roots { + assert!( + reaches(&installable.dag, root, chain), + "root {root} does not read the chain" + ); + } +} + +/// Whether `from` reaches `to` through its inputs. +fn reaches( + dag: &asap_planner::PhysicalASAPDAG, + from: asap_planner::PhysicalASAPNodeId, + to: asap_planner::PhysicalASAPNodeId, +) -> bool { + let mut stack = vec![from]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(node) = stack.pop() { + if node == to { + return true; + } + if seen.insert(node) { + stack.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == node) + .map(|edge| edge.producer), + ); + } + } + false +} + +/// The joint plan's measured ingestion CPU is about one chain's, not two +/// (≤ 1.2 × the single-query plan's). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn shared_node_measured_once() { + use asap_devtools::measurement::benchmark::{bench, BenchArm, BenchRequest, DataSetting}; + use asap_devtools::measurement::calibration_tables::PlanningTables; + let tables = PlanningTables { + cost_inputs: CostInputs::default(), + accuracy: Default::default(), + warnings: Vec::new(), + }; + let setting = DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms: 10_000, + raw_data_retained: false, + }; + let ingestion_cpu = |workload: &PlanningWorkload| { + let report = bench(&BenchRequest { + workload, + arms: vec![BenchArm::NoRaw], + settings: vec![setting.clone()], + series_count: 100, + duration_ms: 600_000, + repetitions: 5, + tables: tables.clone(), + seed: 1, + cost_gate: false, + output_root: None, + }) + .unwrap(); + report.arms[0] + .measured + .as_ref() + .unwrap() + .ingestion_cpu_seconds_per_second + }; + let single = workload( + &[("sum by (job) (sum_over_time(m[5m]))", exact())], + 10_000, + &[("m", facts(100, 1_000))], + ); + let joint = workload( + &[ + ("sum by (job) (sum_over_time(m[5m]))", exact()), + ("sum by (job) (sum_over_time(m[10m]))", exact()), + ], + 10_000, + &[("m", facts(100, 1_000))], + ); + let (single, joint) = (ingestion_cpu(&single), ingestion_cpu(&joint)); + assert!(joint <= 1.2 * single, "joint {joint} vs single {single}"); +} diff --git a/crates/devtools/tests/measurement/cost_fit.rs b/crates/devtools/tests/measurement/cost_fit.rs new file mode 100644 index 000000000..dc43b656e --- /dev/null +++ b/crates/devtools/tests/measurement/cost_fit.rs @@ -0,0 +1,239 @@ +//! §2.1: the cost fit `cpu_ns = units · ns_per_unit + ns_per_invocation` +//! over synthetic measurements of known coefficients, and the measurement +//! loop over a fake clock. + +use std::cell::Cell; +use std::rc::Rc; +use std::time::{Duration, Instant}; + +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::cost_calibration::{ + calibration_cases, fit_operator_cost, measure_kernel, CalibrationFamilies, CalibrationKernel, + CostSample, CpuClock, FitFlag, ThreadCpuClock, +}; + +fn samples(points: &[(f64, f64)]) -> Vec { + points + .iter() + .map(|&(units, cpu_ns)| CostSample { units, cpu_ns }) + .collect() +} + +/// The line through (1e3, 60,000), (1e4, 510,000), (1e5, 5,010,000) is +/// 50 ns per unit + 10,000 ns per invocation. +#[test] +fn fit_recovers_exact_line() { + let fit = fit_operator_cost(&samples(&[ + (1e3, 60_000.0), + (1e4, 510_000.0), + (1e5, 5_010_000.0), + ])); + assert!((fit.ns_per_unit - 50.0).abs() < 1e-9, "{fit:?}"); + assert!( + (fit.ns_per_invocation - 10_000.0).abs() < 1e-9 * 10_000.0, + "{fit:?}" + ); + assert!(fit.flags.is_empty(), "{fit:?}"); +} + +/// The same points × (1.02, 0.98, 1.01). Relative-weighted least squares +/// gives 49.69 / 11,391 (computed by hand from the normal equations with +/// weights 1/y²); plain least squares gives an intercept of 2,783, which +/// fails. +// provisional answer 11 +#[test] +fn fit_is_relative_weighted() { + let fit = fit_operator_cost(&samples(&[ + (1e3, 60_000.0 * 1.02), + (1e4, 510_000.0 * 0.98), + (1e5, 5_010_000.0 * 1.01), + ])); + assert!((48.5..=51.5).contains(&fit.ns_per_unit), "{fit:?}"); + assert!( + (8_000.0..=12_500.0).contains(&fit.ns_per_invocation), + "{fit:?}" + ); +} + +/// Five sizes from 10 to 1e5 units, noise +2%, −2%, +2%, −2%, +2%: the +/// weighted fit is 50.12 / 10,045 by hand, within 1% of 50 / 10,000. +// provisional answer 11 +#[test] +fn fit_five_sizes_tightens() { + let noise = [1.02, 0.98, 1.02, 0.98, 1.02]; + let points: Vec<(f64, f64)> = [10.0, 100.0, 1e3, 1e4, 1e5] + .iter() + .zip(noise) + .map(|(&units, noise)| (units, (50.0 * units + 10_000.0) * noise)) + .collect(); + let fit = fit_operator_cost(&samples(&points)); + assert!((fit.ns_per_unit - 50.0).abs() <= 0.5, "{fit:?}"); + assert!((fit.ns_per_invocation - 10_000.0).abs() <= 100.0, "{fit:?}"); +} + +/// (100, 900), (1000, 10,900) have intercept −211. The fit clamps it to +/// 0, refits through the origin (weighted: Σ(x/y) / Σ(x²/y²) = 9.7702), +/// flags the clamp, and the entry validates. +// provisional answer 11 +#[test] +fn fit_negative_intercept_is_clamped() { + let fit = fit_operator_cost(&samples(&[(100.0, 900.0), (1_000.0, 10_900.0)])); + assert_eq!(fit.ns_per_invocation, 0.0); + assert!( + (fit.ns_per_unit - 9.770_231_720_134_127).abs() < 1e-9, + "{fit:?}" + ); + assert!( + fit.flags.iter().any( + |flag| matches!(flag, FitFlag::NegativeInterceptClamped { raw_intercept_ns } + if (raw_intercept_ns + 211.111).abs() < 0.01) + ), + "{fit:?}" + ); + let inputs = CostInputs { + measured_operation_costs: MeasuredOperationCosts { + operators: vec![OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: fit.ns_per_unit, + ns_per_invocation: fit.ns_per_invocation, + memory_bytes_per_state: None, + }], + ..MeasuredOperationCosts::default() + }, + ..CostInputs::default() + }; + inputs.validate().unwrap(); +} + +/// A quadratic kernel (1e3, 1e4), (1e4, 1e5), (1e5, 1e7) misses a line by +/// far more than 10% at some point (−89% at 1e5 for the weighted line): +/// flagged, not silently accepted. +// provisional answer 11 +#[test] +fn fit_flags_nonlinear() { + let fit = fit_operator_cost(&samples(&[(1e3, 1e4), (1e4, 1e5), (1e5, 1e7)])); + assert!( + fit.flags + .iter() + .any(|flag| matches!(flag, FitFlag::ResidualAboveTolerance { .. })), + "{fit:?}" + ); +} + +/// A clock only the fake kernel advances. +struct FakeClock(Rc>); + +impl CpuClock for FakeClock { + fn thread_cpu_ns(&self) -> u64 { + self.0.get() + } +} + +/// Charges 50 ns per unit + 10 µs per invocation to the fake clock. +struct FakeKernel(Rc>); + +impl CalibrationKernel for FakeKernel { + fn run(&mut self, units: u64) { + self.0.set(self.0.get() + 50 * units + 10_000); + } +} + +/// Driven by the measurement loop at three sizes, a kernel of known cost +/// is recovered exactly: the loop times one invocation per sample, not a +/// batch, and reports the units the kernel processed. +#[test] +fn fit_recovers_fake_clock_kernel() { + let ticks = Rc::new(Cell::new(0)); + let clock = FakeClock(Rc::clone(&ticks)); + let mut kernel = FakeKernel(ticks); + let measured = measure_kernel(&mut kernel, &clock, &[1_000, 10_000, 100_000], 5); + assert_eq!( + measured, + samples(&[(1e3, 60_000.0), (1e4, 510_000.0), (1e5, 5_010_000.0)]) + ); + let fit = fit_operator_cost(&measured); + assert!((fit.ns_per_unit - 50.0).abs() < 1e-9, "{fit:?}"); + assert!((fit.ns_per_invocation - 10_000.0).abs() < 1e-6, "{fit:?}"); +} + +/// Busy-spins 50 ns per unit + 20 µs per invocation of wall time on this +/// thread, which is that much thread CPU time on an idle core. +struct SpinKernel; + +impl CalibrationKernel for SpinKernel { + fn run(&mut self, units: u64) { + let until = Instant::now() + Duration::from_nanos(50 * units + 20_000); + while Instant::now() < until { + std::hint::spin_loop(); + } + } +} + +/// On the thread CPU clock, a spinning kernel's coefficients come back +/// within 10% (per unit) and 25% (per invocation). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn fit_recovers_spin_kernel() { + let measured = measure_kernel( + &mut SpinKernel, + &ThreadCpuClock, + &[1_000, 10_000, 100_000], + 9, + ); + let fit = fit_operator_cost(&measured); + assert!((fit.ns_per_unit - 50.0).abs() <= 5.0, "{fit:?}"); + assert!( + (fit.ns_per_invocation - 20_000.0).abs() <= 5_000.0, + "{fit:?}" + ); +} + +/// Each case's microbenchmark counts the units the planner prices it by: +/// pricing the case's DAG with only the case's entry at 1 ns per unit and +/// none per invocation, the planner's measured node reports exactly the +/// microbenchmark's unit count (a scan's output rows, a merge's merged +/// states, any other node's first input rows). +#[test] +fn units_match_cost_model() { + for case in calibration_cases(&CalibrationFamilies::All) { + let calibration_dag = case.dag_at(1_000); + let fit = asap_devtools::measurement::cost_calibration::FittedOperatorCost { + ns_per_unit: 1.0, + ns_per_invocation: 0.0, + flags: Vec::new(), + }; + let inputs = CostInputs { + measured_operation_costs: MeasuredOperationCosts { + operators: vec![case.operator_cost(&fit, None)], + ..MeasuredOperationCosts::default() + }, + ..CostInputs::default() + }; + let (cost, _) = asap_cost::plan_cost( + &calibration_dag.dag, + &calibration_dag.entries, + &calibration_dag.data, + &inputs, + ) + .unwrap(); + let node = &cost.per_node[&calibration_dag.measured_node]; + assert_eq!( + node.cost_source, + asap_cost::NodeCostSource::Measured, + "{case:?}" + ); + // The planner writes "measured x ns + ns". + let units: u64 = node + .detail + .split("measured ") + .nth(1) + .and_then(|rest| rest.split(' ').next()) + .and_then(|units| units.parse().ok()) + .unwrap_or_else(|| panic!("{case:?}: {}", node.detail)); + assert_eq!(units, calibration_dag.units, "{case:?}"); + } +} diff --git a/crates/devtools/tests/measurement/end_to_end.rs b/crates/devtools/tests/measurement/end_to_end.rs new file mode 100644 index 000000000..905acdf24 --- /dev/null +++ b/crates/devtools/tests/measurement/end_to_end.rs @@ -0,0 +1,349 @@ +//! §2.12 and §2.13: the `asap_run` binary end to end on a tiny workload, +//! the showcase suite, and determinism under a seed. + +use std::process::Command; + +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, ZipfParameter}; +use asap_devtools::measurement::benchmark::{bench_showcase, GateVerdict}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest, PlanningTables}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::measurement::samples::{generate_samples, GeneratedDataSettings}; +use asap_devtools::measurement::source_facts::{source_facts_of_samples, FactsOptions}; + +use super::common::{exact, four_constant_series, four_series_facts, workload, TempDir}; + +fn asap_run(args: &[&str], dir: &TempDir) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_asap_run")) + .args(args) + .env("ASAP_CALIBRATION_DIR", &dir.0) + .current_dir(&dir.0) + .output() + .unwrap() +} + +fn jsonl(samples: &[asap_devtools::measurement::samples::Sample]) -> String { + samples + .iter() + .map(|sample| serde_json::to_string(sample).unwrap() + "\n") + .collect() +} + +/// `facts` on the hand-made JSONL, `calibrate --small --families exact`, +/// then `bench` on `sum by (job) (sum_over_time(m[1m]))`: the facts are the +/// data's, the costs validate, the bench answers 180 / 420, the gate +/// passes on answers (`--no-cost-gate`) and the run's files are written. +#[test] +fn e2e_tiny_hand_answers() { + let dir = TempDir::new("e2e"); + let data = dir.write( + "m.jsonl", + &jsonl(&four_constant_series(600_000, |_, _| None)), + ); + let plain = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[], + ); + let workload_path = dir.write("w.json", &serde_json::to_string(&plain).unwrap()); + let (data, workload_path) = (data.to_str().unwrap(), workload_path.to_str().unwrap()); + + let facts = asap_run( + &["facts", "--data", data, "--workload", workload_path], + &dir, + ); + assert!( + facts.status.success(), + "{}", + String::from_utf8_lossy(&facts.stderr) + ); + let with_facts: asap_planner::PlanningWorkload = + serde_json::from_slice(&facts.stdout).expect("facts writes the workload"); + let m = &with_facts.data_workload.as_ref().unwrap().sources["m"]; + assert_eq!(m.series_count, Some(4)); + assert_eq!(m.sample_interval.map(|interval| interval.0), Some(1_000)); + + let calibrated = asap_run(&["calibrate", "--small", "--families", "exact"], &dir); + assert!( + calibrated.status.success(), + "{}", + String::from_utf8_lossy(&calibrated.stderr) + ); + let costs = std::fs::read_dir(&dir.0) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path().join("costs.json")) + .find(|path| path.is_file()) + .expect("calibrate wrote /costs.json"); + let table: asap_cost::MeasuredOperationCosts = + serde_json::from_str(&std::fs::read_to_string(costs).unwrap()).unwrap(); + asap_cost::CostInputs { + measured_operation_costs: table, + ..asap_cost::CostInputs::default() + } + .validate() + .unwrap(); + + let facts_path = dir.write("w-facts.json", &serde_json::to_string(&with_facts).unwrap()); + let executed = asap_run( + &[ + "execute", + "--workload", + facts_path.to_str().unwrap(), + "--data", + data, + "--at", + "600000", + ], + &dir, + ); + assert!( + executed.status.success(), + "{}", + String::from_utf8_lossy(&executed.stderr) + ); + let answers = String::from_utf8_lossy(&executed.stdout); + assert!( + answers.contains("180") && answers.contains("420"), + "{answers}" + ); + + let benched = asap_run( + &[ + "bench", + "--workload", + facts_path.to_str().unwrap(), + "--arms", + "exact,planned,noraw", + "--duration", + "600000", + "--no-cost-gate", + ], + &dir, + ); + assert!( + benched.status.success(), + "{}", + String::from_utf8_lossy(&benched.stderr) + ); + let runs = dir.0.join("runs"); + assert!(runs.is_dir(), "bench wrote no runs/ directory"); +} + +/// `bench --suite showcase --small`: every row passes (from milestone 1). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn e2e_small_showcase() { + let report = bench_showcase(true, &calibrated(true), None).unwrap(); + assert_eq!(report.verdict, GateVerdict::Pass, "{:?}", report.rows); +} + +/// The full showcase meets the milestone criteria. +#[test] +#[ignore = "milestone scale"] +fn e2e_full_showcase() { + let report = bench_showcase(false, &calibrated(false), None).unwrap(); + assert_eq!(report.verdict, GateVerdict::Pass, "{:?}", report.rows); +} + +fn calibrated(small: bool) -> PlanningTables { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small, + seed: 7, + }) + .unwrap(); + PlanningTables { + cost_inputs: asap_cost::CostInputs { + measured_operation_costs: calibration.costs, + ..asap_cost::CostInputs::default() + }, + accuracy: calibration.accuracy, + warnings: Vec::new(), + } +} + +/// A hash of every sample `description` generates. +fn stream_hash(description: &TimeSeriesDescription) -> u64 { + use std::hash::{Hash, Hasher}; + let generated = description.generate().unwrap(); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for sample in generated.samples() { + let (labels, ts, value) = sample.unwrap(); + labels.hash(&mut hasher); + ts.hash(&mut hasher); + value.to_bits().hash(&mut hasher); + } + hasher.finish() +} + +/// `aqpbm-datagen`: one description and seed give one sample stream; a +/// different seed another. +#[test] +fn datagen_seeded_bit_identical() { + let description = TimeSeriesDescription { + labels: vec![ + SeriesLabelSpec { + name: "job".into(), + cardinality: 4, + skewness: 0.0, + }, + SeriesLabelSpec { + name: "instance".into(), + cardinality: 16, + skewness: 1.1, + }, + ], + series_num: 8, + start_ms: 0, + sample_interval_ms: 1_000, + duration_ms: 600_000, + value_distribution: DataDistribution::Zipf(ZipfParameter { + skewness: 1.1, + population_size: 1_000, + seed: 0, + }), + counter: false, + seed: 42, + }; + assert_eq!(stream_hash(&description), stream_hash(&description)); + let other = TimeSeriesDescription { + seed: 43, + ..description.clone() + }; + assert_ne!(stream_hash(&description), stream_hash(&other)); +} + +/// Data generated from the facts is identical across runs under one seed. +#[test] +fn generated_samples_seeded_bit_identical() { + let plain = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let data = plain.data_workload.clone().unwrap(); + let settings = GeneratedDataSettings { + seed: 42, + start_ms: 0, + duration_ms: 600_000, + }; + assert_eq!( + generate_samples(&data, &settings).unwrap(), + generate_samples(&data, &settings).unwrap() + ); +} + +/// `facts` twice gives byte-identical JSON. +#[test] +fn facts_seeded_bit_identical() { + let samples = four_constant_series(600_000, |series, ts| { + (series == 1).then_some((ts % 13_000) as f64) + }); + let plain = workload( + &[("sum by (job, instance) (sum_over_time(m[1m]))", exact())], + 10_000, + &[], + ); + let once = || { + let report = source_facts_of_samples(&samples, &plain, FactsOptions::default()).unwrap(); + serde_json::to_string(&report.data_workload).unwrap() + }; + assert_eq!(once(), once()); +} + +/// Two small calibrations write byte-identical accuracy tables (trials +/// are seeded). +#[test] +fn calibrate_accuracy_seeded() { + let once = || { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: true, + seed: 7, + }) + .unwrap(); + serde_json::to_string(&calibration.accuracy).unwrap() + }; + assert_eq!(once(), once()); +} + +/// The same tables give the same `plan.txt`. +#[test] +fn plan_given_tables_identical() { + let dir = TempDir::new("plan_txt"); + let plain = workload( + &[("sum by (job) (sum_over_time(m[1m]))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let workload_path = dir.write("w.json", &serde_json::to_string(&plain).unwrap()); + for _ in 0..2 { + let output = asap_run( + &[ + "bench", + "--workload", + workload_path.to_str().unwrap(), + "--arms", + "planned", + "--duration", + "600000", + "--no-cost-gate", + ], + &dir, + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let runs = dir.0.join("runs"); + let machine = std::fs::read_dir(&runs) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let plans: Vec = std::fs::read_dir(machine) + .unwrap() + .map(|run| std::fs::read_to_string(run.unwrap().path().join("plan.txt")).unwrap()) + .collect(); + assert_eq!(plans.len(), 2, "two runs, two run directories"); + assert_eq!(plans[0], plans[1]); +} + +/// Two full calibrations: the costs differ by at most the noise and the +/// showcase plan selection is identical, or each change still passes the +/// gate. +// provisional answer 23 +#[test] +#[ignore = "milestone scale"] +fn plan_stable_under_recalibration() { + let first = calibrated(false); + let second = calibrated(false); + for (a, b) in first + .cost_inputs + .measured_operation_costs + .operators + .iter() + .zip(&second.cost_inputs.measured_operation_costs.operators) + { + assert_eq!( + (a.operator, &a.phase, &a.sketch, &a.exact, &a.measure), + (b.operator, &b.phase, &b.sketch, &b.exact, &b.measure) + ); + let ratio = a.ns_per_unit.max(1e-9) / b.ns_per_unit.max(1e-9); + assert!((1.0 / 1.05..=1.05).contains(&ratio), "{a:?} vs {b:?}"); + } + let first_run = bench_showcase(false, &first, None).unwrap(); + let second_run = bench_showcase(false, &second, None).unwrap(); + for (a, b) in first_run.arms.iter().zip(&second_run.arms) { + if a.plan != b.plan { + assert_eq!( + second_run.verdict, + GateVerdict::Pass, + "a changed plan fails the gate" + ); + } + } +} diff --git a/crates/devtools/tests/measurement/execution.rs b/crates/devtools/tests/measurement/execution.rs new file mode 100644 index 000000000..fd423ad27 --- /dev/null +++ b/crates/devtools/tests/measurement/execution.rs @@ -0,0 +1,373 @@ +//! §2.7: the reference driver over hand-made data. Four 1 Hz series of +//! `m`, constant 1 and 2 (`job="a"`) and 3 and 4 (`job="b"`); every +//! expected answer is counted by hand over the window `(t − range, t]`. + +use std::sync::mpsc; +use std::time::Duration; + +use asap_devtools::measurement::execution::{ + execute, DataInput, EvaluationAnswers, ExecuteError, ExecuteRequest, ExecutionReport, + RootDrainOrder, +}; +use asap_devtools::measurement::samples::Sample; +use asap_planner::{DeploymentCapabilities, MaterializedViewRefresh, PlanningWorkload}; + +use super::common::{ + exact, exact_match, facts, four_constant_series, four_series_facts, job, no_raw_data, + per_value, sample, workload, +}; + +const SUM_BY_JOB: &str = "sum by (job) (sum_over_time(m[1m]))"; + +fn sum_by_job() -> PlanningWorkload { + workload( + &[(SUM_BY_JOB, exact())], + 10_000, + &[("m", four_series_facts())], + ) +} + +fn run(request: &ExecuteRequest<'_>) -> ExecutionReport { + execute(request).unwrap() +} + +/// Root `root`'s value for `job` at evaluation `evaluation`. +fn value(report: &ExecutionReport, evaluation: usize, root: usize, name: &str) -> f64 { + report.evaluations[evaluation].per_root[root].values[&job(name)] +} + +/// At t = 600 s the window (540 s, 600 s] holds 60 samples per series: +/// a: 60 · (1 + 2) = 180, b: 60 · (3 + 4) = 420. A window closed on both +/// ends would hold 61 (183 / 427). +#[test] +fn execute_sum_over_time_hand() { + let workload = sum_by_job(); + let request = ExecuteRequest::new( + &workload, + DataInput::Samples(four_constant_series(600_000, |_, _| None)), + vec![600_000], + ); + let report = run(&request); + assert_eq!(value(&report, 0, 0, "a"), 180.0); + assert_eq!(value(&report, 0, 0, "b"), 420.0); + assert_eq!(report.evaluations[0].per_root[0].values.len(), 2); +} + +/// Series 1's sample at exactly t − 60 s = 540 s is 1000: excluded, so a +/// is still 180; the sample at t = 600 s is included (else 177). +#[test] +fn execute_window_boundary() { + let data = four_constant_series(600_000, |series, ts| { + (series == 0 && ts == 540_000).then_some(1_000.0) + }); + let workload = sum_by_job(); + let request = ExecuteRequest::new(&workload, DataInput::Samples(data), vec![600_000]); + let report = run(&request); + assert_eq!(value(&report, 0, 0, "a"), 180.0); +} + +/// A counter `c_total` = t / 1000 on every series rises 1 per second: +/// `sum by (job) (rate(c_total[1m]))` is 2.0 for each job. +#[test] +fn execute_rate_linear_counter() { + let mut samples = Vec::new(); + for ts in (0..=600_000).step_by(1_000) { + for (job, instance) in [("a", "1"), ("a", "2"), ("b", "3"), ("b", "4")] { + samples.push(sample( + "c_total", + &[("job", job), ("instance", instance)], + ts, + ts as f64 / 1000.0, + )); + } + } + let workload = workload( + &[("sum by (job) (rate(c_total[1m]))", exact())], + 10_000, + &[("c_total", four_series_facts())], + ); + let report = run(&ExecuteRequest::new( + &workload, + DataInput::Samples(samples), + vec![600_000], + )); + assert!(exact_match(value(&report, 0, 0, "a"), 2.0), "{report:?}"); + assert!(exact_match(value(&report, 0, 0, "b"), 2.0), "{report:?}"); +} + +/// Series 1 changes to 10 after t = 605 s. At 600 s: a = 180; at 610 s the +/// window holds 5 samples of 10: 180 − 5 + 50 = 225; at 620 s, 15: +/// 180 − 15 + 150 = 315. b stays 420. +#[test] +fn execute_several_eval_times() { + let data = four_constant_series(620_000, |series, ts| { + (series == 0 && ts > 605_000).then_some(10.0) + }); + let workload = sum_by_job(); + let request = ExecuteRequest::new( + &workload, + DataInput::Samples(data), + vec![600_000, 610_000, 620_000], + ); + let report = run(&request); + let a: Vec = (0..3).map(|i| value(&report, i, 0, "a")).collect(); + assert_eq!(a, vec![180.0, 225.0, 315.0]); + assert!((0..3).all(|i| value(&report, i, 0, "b") == 420.0)); +} + +/// The same query planned at query time over raw data and as an +/// ingestion-time chain (no raw data) answers the same. +// provisional answer 21 +#[test] +fn execute_ingestion_equals_query_time() { + let data = four_constant_series(620_000, |series, ts| { + (series == 0 && ts > 605_000).then_some(10.0) + }); + let workload = sum_by_job(); + let times = vec![600_000, 610_000, 620_000]; + let query_time = run(&ExecuteRequest::new( + &workload, + DataInput::Samples(data.clone()), + times.clone(), + )); + let mut chained = ExecuteRequest::new(&workload, DataInput::Samples(data), times); + chained.deployment_capabilities = no_raw_data(); + let chained = run(&chained); + for (left, right) in query_time.evaluations.iter().zip(&chained.evaluations) { + for (l, r) in left.per_root.iter().zip(&right.per_root) { + assert_eq!( + l.values.keys().collect::>(), + r.values.keys().collect::>() + ); + for (key, value) in &l.values { + assert!( + exact_match(*value, r.values[key]), + "{key:?}: {value} vs {}", + r.values[key] + ); + } + } + } +} + +/// Two queries over one producer: `sum` and `count` of each job's +/// samples in the last minute. +fn joint_workload() -> PlanningWorkload { + workload( + &[ + (SUM_BY_JOB, exact()), + ("sum by (job) (count_over_time(m[1m]))", exact()), + ], + 10_000, + &[("m", four_series_facts())], + ) +} + +/// Run `workload` over `data` at t = 600 s with `order` on another thread +/// and return its answers, failing after 10 s: a deadlock guard, not a +/// timing assertion. +fn answers_with_deadlock_guard( + workload: &'static PlanningWorkload, + data: Vec, + order: RootDrainOrder, +) -> Vec { + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + let mut request = ExecuteRequest::new(workload, DataInput::Samples(data), vec![600_000]); + request.root_drain_order = order; + let _ = sender.send(execute(&request).map(|report| report.evaluations)); + }); + receiver + .recv_timeout(Duration::from_secs(10)) + .expect("the run did not finish within 10 s") + .unwrap() +} + +/// Draining the roots of a joint plan in any order gives the same answers +/// and finishes: sum 180 / 420, count 120 / 120. +#[test] +fn execute_drain_order() { + let workload: &'static PlanningWorkload = Box::leak(Box::new(joint_workload())); + let data = four_constant_series(600_000, |_, _| None); + let mut answers = Vec::new(); + for order in [ + RootDrainOrder::Reversed, + RootDrainOrder::WorkloadOrder, + RootDrainOrder::Concurrent, + ] { + let evaluations = answers_with_deadlock_guard(workload, data.clone(), order); + assert_eq!(evaluations[0].per_root[0].values[&job("a")], 180.0); + assert_eq!(evaluations[0].per_root[1].values[&job("a")], 120.0); + assert_eq!(evaluations[0].per_root[1].values[&job("b")], 120.0); + answers.push(evaluations); + } + assert!(answers.windows(2).all(|pair| pair[0] == pair[1])); +} + +/// Without raw data both queries read ingestion-time views fed by one +/// scan: at ingestion time the scan runs once per window and each batch +/// reaches both chain builds, so both views are complete (sum 180 / 420, +/// count 120 / 120), not one view missing the batches the other read first. +#[test] +fn execute_shared_ingestion_scan_feeds_every_consumer() { + let mut request = ExecuteRequest::new( + Box::leak(Box::new(joint_workload())), + DataInput::Samples(four_constant_series(600_000, |_, _| None)), + vec![600_000], + ); + request.deployment_capabilities = no_raw_data(); + let report = run(&request); + assert_eq!(value(&report, 0, 0, "a"), 180.0); + assert_eq!(value(&report, 0, 0, "b"), 420.0); + assert_eq!(value(&report, 0, 1, "a"), 120.0); + assert_eq!(value(&report, 0, 1, "b"), 120.0); +} + +/// A 1m window over a 10 s chain evaluated 20 times every 10 s: after the +/// warm-up the view holds a constant number of states, 6 windows × 2 +/// groups. +#[test] +fn execute_retention_evicts() { + let times: Vec = (0..20).map(|i| 600_000 + i * 10_000).collect(); + let mut request = ExecuteRequest::new( + Box::leak(Box::new(sum_by_job())), + DataInput::Samples(four_constant_series(*times.last().unwrap(), |_, _| None)), + times, + ); + request.deployment_capabilities = no_raw_data(); + let report = run(&request); + let chain = report + .views + .iter() + .find(|view| view.refresh == MaterializedViewRefresh::AtIngestion) + .expect("an ingestion-time view"); + let after_warm_up = &chain.states_after_evaluation[1..]; + assert!( + after_warm_up.iter().all(|&states| states == 12), + "{after_warm_up:?}" + ); +} + +/// An evaluation at t = 30 s, before a 1m window of data exists, is +/// refused, never answered from half a window. +// provisional answer 18 +#[test] +fn execute_before_warmup() { + let workload = sum_by_job(); + let request = ExecuteRequest::new( + &workload, + DataInput::Samples(four_constant_series(600_000, |_, _| None)), + vec![30_000], + ); + assert!(matches!( + execute(&request), + Err(ExecuteError::BeforeWarmUp { at_ms: 30_000, .. }) + )); +} + +/// A plan with a view refreshed on first read (query-time results kept +/// across evaluations) is run, not refused: the view is refreshed once +/// and the second evaluation reuses it. +#[test] +fn execute_on_first_read_view() { + let workload = workload( + &[("sum by (job) (sum_over_time(m[1m] offset 1m))", exact())], + 10_000, + &[("m", four_series_facts())], + ); + let mut request = ExecuteRequest::new( + Box::leak(Box::new(workload)), + DataInput::Samples(four_constant_series(620_000, |_, _| None)), + vec![610_000, 620_000], + ); + request.deployment_capabilities = DeploymentCapabilities { + query_time_results_retained: true, + ..DeploymentCapabilities::default() + }; + let report = run(&request); + let view = report + .views + .iter() + .find(|view| view.refresh == MaterializedViewRefresh::OnFirstRead) + .expect("the plan has an on-first-read view"); + assert_eq!(view.refresh_count, 1, "{view:?}"); + assert_eq!(value(&report, 1, 0, "a"), 180.0); +} + +/// `quantile_over_time(0.5, m[1m])` over values 1 … 60: exactly, 30.5 (the +/// interpolated median); a DDSketch with α = 0.01 answers within 1% of the +/// nearest-rank bracket [30, 31]. +// provisional answer 4 +#[test] +fn execute_quantile_reference() { + // Values 1 … 60 in the window (540 s, 600 s]. + let samples: Vec = (0..=600) + .map(|second| { + let value = if second > 540 { + (second - 540) as f64 + } else { + 1.0 + }; + sample("lat", &[("job", "a")], second * 1_000, value) + }) + .collect(); + let query = "quantile_over_time(0.5, lat[1m])"; + let exact_workload = workload(&[(query, exact())], 10_000, &[("lat", facts(1, 1_000))]); + let exact_report = run(&ExecuteRequest::new( + &exact_workload, + DataInput::Samples(samples.clone()), + vec![600_000], + )); + assert_eq!(value(&exact_report, 0, 0, "a"), 30.5); + let sketch_workload = workload( + &[(query, per_value("relative", 0.01, 0.0))], + 10_000, + &[("lat", facts(1, 1_000))], + ); + let sketch_report = run(&ExecuteRequest::new( + &sketch_workload, + DataInput::Samples(samples), + vec![600_000], + )); + let answer = value(&sketch_report, 0, 0, "a"); + assert!((30.0 * 0.99..=31.0 * 1.01).contains(&answer), "{answer}"); +} + +/// Two runs, and two drain orders, give identical answer maps (I9). +#[test] +fn execute_answers_bit_identical() { + let workload: &'static PlanningWorkload = Box::leak(Box::new(joint_workload())); + let data = four_constant_series(620_000, |series, ts| { + (series == 2 && ts % 7_000 == 0).then_some(0.1 * ts as f64) + }); + let mut runs = Vec::new(); + for order in [ + RootDrainOrder::WorkloadOrder, + RootDrainOrder::WorkloadOrder, + RootDrainOrder::Reversed, + ] { + let mut request = ExecuteRequest::new( + workload, + DataInput::Samples(data.clone()), + vec![600_000, 620_000], + ); + request.root_drain_order = order; + runs.push(run(&request).evaluations); + } + for pair in runs.windows(2) { + for (left, right) in pair[0].iter().zip(&pair[1]) { + for (l, r) in left.per_root.iter().zip(&right.per_root) { + assert_eq!( + l.values + .iter() + .map(|(k, v)| (k.clone(), v.to_bits())) + .collect::>(), + r.values + .iter() + .map(|(k, v)| (k.clone(), v.to_bits())) + .collect::>() + ); + } + } + } +} diff --git a/crates/devtools/tests/measurement/gate.rs b/crates/devtools/tests/measurement/gate.rs new file mode 100644 index 000000000..862d11868 --- /dev/null +++ b/crates/devtools/tests/measurement/gate.rs @@ -0,0 +1,298 @@ +//! §2.9: the bench gate's rule, and runs whose tables lie. + +use std::path::PathBuf; + +use asap_accuracy::{ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy}; +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::benchmark::{ + bench, gate_verdict, BenchArm, BenchRequest, DataSetting, GateFailure, GateRow, GateVerdict, +}; +use asap_devtools::measurement::calibration_tables::PlanningTables; +use asap_planner::PlanningWorkload; +use asap_types::accuracy::AccuracyMetric; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +use super::common::{facts, per_value, workload, TempDir}; + +fn setting() -> DataSetting { + DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms: 10_000, + raw_data_retained: true, + } +} + +fn row(exact: Option, selected: Result, error: f64, target: f64) -> GateRow { + GateRow { + workload: "w".into(), + setting: setting(), + exact_cost_per_second: exact, + selected_cost_per_second: selected, + measured_error: error, + target_error: target, + } +} + +fn verdict_of(row: GateRow) -> GateVerdict { + gate_verdict(&[("w".into(), setting())], &[row], true) +} + +fn fails(verdict: &GateVerdict) -> bool { + matches!(verdict, GateVerdict::Fail(failures) if !failures.is_empty()) +} + +/// The threshold arithmetic: cost 105 against exact 100 passes (≤ 1.05 ×), +/// 105.01 fails; error 0.0100 against 0.01 passes (`within_limit`), 0.0101 +/// fails; a missing exact arm, a failed selected arm, a NaN error, and a +/// missing or extra row each fail. +// provisional answer 1 +#[test] +fn gate_rule_pure() { + assert_eq!( + verdict_of(row(Some(100.0), Ok(105.0), 0.0, 0.01)), + GateVerdict::Pass + ); + assert!(fails(&verdict_of(row(Some(100.0), Ok(105.01), 0.0, 0.01)))); + assert_eq!( + verdict_of(row(Some(100.0), Ok(50.0), 0.0100, 0.01)), + GateVerdict::Pass + ); + assert!(fails(&verdict_of(row(Some(100.0), Ok(50.0), 0.0101, 0.01)))); + assert!(matches!( + verdict_of(row(None, Ok(50.0), 0.0, 0.01)), + GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::ExactArmMissing { .. })) + )); + assert!(matches!( + verdict_of(row(Some(100.0), Err("panicked".into()), 0.0, 0.01)), + GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::SelectedArmFailed { .. })) + )); + assert!(matches!( + verdict_of(row(Some(100.0), Ok(50.0), f64::NAN, 0.01)), + GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::ErrorNotANumber { .. })) + )); + let missing = gate_verdict( + &[("w".into(), setting()), ("v".into(), setting())], + &[row(Some(100.0), Ok(50.0), 0.0, 0.01)], + true, + ); + assert!( + matches!(&missing, GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::MissingRow { .. }))) + ); + let extra = gate_verdict(&[], &[row(Some(100.0), Ok(50.0), 0.0, 0.01)], true); + assert!( + matches!(&extra, GateVerdict::Fail(failures) if failures.iter().any(|f| matches!(f, GateFailure::ExtraRow { .. }))) + ); + // Without the cost gate, a costly plan with a met target passes. + assert_eq!( + gate_verdict( + &[("w".into(), setting())], + &[row(Some(100.0), Ok(500.0), 0.0, 0.01)], + false + ), + GateVerdict::Pass + ); +} + +fn tables(costs: MeasuredOperationCosts, accuracy: MeasuredSketchAccuracy) -> PlanningTables { + PlanningTables { + cost_inputs: CostInputs { + measured_operation_costs: costs, + ..CostInputs::default() + }, + accuracy, + warnings: Vec::new(), + } +} + +fn request<'a>( + workload: &'a PlanningWorkload, + tables: PlanningTables, + output_root: Option, +) -> BenchRequest<'a> { + BenchRequest { + workload, + arms: vec![BenchArm::Exact, BenchArm::Planned], + settings: vec![setting()], + series_count: 100, + duration_ms: 600_000, + repetitions: 5, + tables, + seed: 1, + cost_gate: true, + output_root, + } +} + +/// A cost table claiming the exact scan costs 1e4 ns per row and the +/// DDSketch build 1e-3 ns per row makes the planner pick the DDSketch +/// query-time build, which measures about 2× the exact plan (217 against +/// 82–107 ns per row in the feasibility run): the gate fails. +// provisional answer 1 and provisional answer 23 +#[test] +#[ignore = "timing: run on a dedicated host"] +fn gate_fails_on_lying_cost_table() { + let workload = workload( + &[( + "quantile_over_time(0.99, m[10m])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("m", facts(100, 1_000))], + ); + let lie = |operator, sketch, ns_per_unit| OperatorCost { + operator, + phase: None, + sketch, + exact: None, + measure: None, + ns_per_unit, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }; + let costs = MeasuredOperationCosts { + operators: vec![ + lie(OperatorKind::Scan, None, 1e4), + lie( + OperatorKind::SummaryAgg, + Some(SketchParams::DDSketch { alpha: 0.01 }), + 1e-3, + ), + ], + ..MeasuredOperationCosts::default() + }; + let report = bench(&request( + &workload, + tables(costs, MeasuredSketchAccuracy::default()), + None, + )) + .unwrap(); + assert!( + matches!(&report.verdict, GateVerdict::Fail(failures) + if failures.iter().any(|f| matches!(f, GateFailure::CostAboveLimit { .. }))), + "{:?}", + report.verdict + ); +} + +/// An accuracy table claiming KLL k = 26 has rank error 1e-4 makes the +/// planner pick k = 26 for rank target 0.005 at δ = 0.01; its measured +/// rank error is about 0.05–0.1: the gate fails on error (deterministic +/// under a seed). +// provisional answer 3 +#[test] +fn gate_fails_on_lying_accuracy_table() { + let workload = workload( + &[( + "quantile_over_time(0.99, m[10m])", + per_value("rank", 0.005, 0.01), + )], + 10_000, + &[("m", facts(100, 1_000))], + ); + let accuracy = MeasuredSketchAccuracy { + entries: vec![MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k: 26 }, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric: AccuracyMetric::Rank, + trials: 1_000, + error_at: vec![ErrorAtFailureProbability { + failure_probability: 0.01, + error: 1e-4, + }], + }], + }; + let mut request = request( + &workload, + tables(MeasuredOperationCosts::default(), accuracy), + None, + ); + request.cost_gate = false; + let report = bench(&request).unwrap(); + let planned = report + .arms + .iter() + .find(|arm| arm.arm == BenchArm::Planned) + .unwrap(); + assert!( + planned + .plan + .dag + .nodes + .iter() + .any(|node| node.parameters == Some(SketchParams::Kll { k: 26 })), + "the lying table did not select k = 26" + ); + assert!( + matches!(&report.verdict, GateVerdict::Fail(failures) + if failures.iter().any(|f| matches!(f, GateFailure::ErrorAboveTarget { .. }))), + "{:?}", + report.verdict + ); +} + +/// The 1m-window counterexample: `quantile_over_time(0.99, m[1m])` at 1 Hz +/// every 10 s selects exact, or a plan within 1.05× of it: the gate passes. +// provisional answer 23 +#[test] +#[ignore = "timing: run on a dedicated host"] +fn gate_counterexample_passes_by_exact() { + let workload = workload( + &[( + "quantile_over_time(0.99, m[1m])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("m", facts(100, 1_000))], + ); + let report = bench(&request( + &workload, + tables( + MeasuredOperationCosts::default(), + MeasuredSketchAccuracy::default(), + ), + None, + )) + .unwrap(); + assert_eq!(report.verdict, GateVerdict::Pass); +} + +/// A run writes `runs///{workload.json, plan.txt, +/// results.csv, summary.md}`, one results row per workload and setting. +#[test] +fn gate_output_files() { + let root = TempDir::new("runs"); + let workload = workload( + &[( + "sum by (job) (sum_over_time(m[1m]))", + super::common::exact(), + )], + 10_000, + &[("m", super::common::four_series_facts())], + ); + let mut request = request( + &workload, + tables( + MeasuredOperationCosts::default(), + MeasuredSketchAccuracy::default(), + ), + Some(root.0.clone()), + ); + request.series_count = 4; + request.cost_gate = false; + let report = bench(&request).unwrap(); + let run_dir = report.run_dir.expect("the run was written"); + assert!(run_dir.starts_with(root.0.join("runs")), "{run_dir:?}"); + assert_eq!( + run_dir.parent().unwrap().parent().unwrap(), + root.0.join("runs") + ); + for file in ["workload.json", "plan.txt", "results.csv", "summary.md"] { + assert!( + run_dir.join(file).is_file(), + "{file} missing in {run_dir:?}" + ); + } + let results = std::fs::read_to_string(run_dir.join("results.csv")).unwrap(); + // A header and one row for one workload at one setting. + assert_eq!(results.lines().count(), 2, "{results}"); +} diff --git a/crates/devtools/tests/measurement/machine_and_tables.rs b/crates/devtools/tests/measurement/machine_and_tables.rs new file mode 100644 index 000000000..a44cf0608 --- /dev/null +++ b/crates/devtools/tests/measurement/machine_and_tables.rs @@ -0,0 +1,367 @@ +//! §2.5: the machine key, which tables a run reads, and the fallback to +//! the formulas. + +use std::path::Path; +use std::process::Command; + +use asap_accuracy::NoMeasuredGuarantees; +use asap_cost::{CostInputs, MeasuredOperationCosts, OperatorCost, OperatorKind}; +use asap_devtools::measurement::calibration_tables::{ + load_planning_tables, resolve_table_paths, TableOverrides, TablePath, CALIBRATION_DIR_ENV, +}; +use asap_devtools::measurement::machine::MachineDescription; +use asap_planner::DeploymentCapabilities; + +use super::common::{exact, facts, plan, workload, TempDir}; + +const XEON_CPUINFO: &str = "processor\t: 0\nvendor_id\t: GenuineIntel\n\ +model name\t: Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz\n\n"; + +/// `/proc/cpuinfo` of `cores` logical cores of `model`. +fn cpuinfo(model: &str, cores: u32) -> String { + (0..cores) + .map(|core| format!("processor\t: {core}\nmodel name\t: {model}\n\n")) + .collect() +} + +fn meminfo(kib: u64) -> String { + format!("MemTotal: {kib} kB\nMemFree: 1000 kB\n") +} + +fn xeon(memory_kib: u64) -> MachineDescription { + MachineDescription::from_proc( + &cpuinfo("Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz", 56), + &meminfo(memory_kib), + "Linux 5.15", + ) + .unwrap() +} + +/// 56 logical cores of a Xeon Gold 6142 with 263,724,000 KiB: +/// 263,724,000 / 2^20 = 251.5 GiB, floored to 251. +// provisional answer 16 +#[test] +fn machine_key_from_cpu_cores_memory() { + assert!(XEON_CPUINFO.contains("6142")); + let machine = xeon(263_724_000); + assert_eq!(machine.logical_cores, 56); + assert_eq!(machine.memory_kib, 263_724_000); + assert_eq!(machine.machine_key(), "xeon-gold-6142-56c-251g"); +} + +/// The key is a function of CPU model, cores and memory only: this +/// machine's detected key contains no token of its hostname that the CPU +/// model does not also contain, and the description has no hostname to +/// change. +#[test] +fn machine_key_ignores_hostname() { + let fixture = xeon(263_724_000).machine_key(); + for hostname in ["a", "xeon-gold-6142", "node-17"] { + // The fixture's inputs do not include a hostname, so every + // hostname gives the same key, and it is never the hostname. + assert_ne!(fixture, hostname); + } + let detected = MachineDescription::detect().unwrap(); + let key = detected.machine_key(); + let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default(); + let model = detected.cpu_model.to_lowercase(); + for token in hostname + .trim() + .to_lowercase() + .split(|c: char| !c.is_ascii_alphanumeric()) + { + if token.len() >= 3 && !model.contains(token) { + assert!( + !key.contains(token), + "key {key} contains hostname token {token}" + ); + } + } +} + +/// `MemTotal` jitters by about ±0.5% across boots: the key does not change. +// provisional answer 16: GiB rounding cannot hold this for the 251.5 GiB +// fixture (+0.5% is 252.8 GiB, −0.5% is 250.2 GiB); the test is written +// as the plan states it so the conflict is visible. +#[test] +fn machine_key_stable_under_memtotal_jitter() { + let key = xeon(263_724_000).machine_key(); + assert_eq!(xeon(263_724_000 * 1005 / 1000).machine_key(), key); + assert_eq!(xeon(263_724_000 * 995 / 1000).machine_key(), key); +} + +/// Only `[a-z0-9-]`, no leading, trailing or double dashes. +#[test] +fn machine_key_slug_charset() { + let machine = MachineDescription::from_proc( + &cpuinfo("AMD EPYC(TM) 7452 32-Core Processor @ 2.35GHz (R)", 8), + &meminfo(16_000_000), + "Linux", + ) + .unwrap(); + let key = machine.machine_key(); + assert!( + key.chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-'), + "{key}" + ); + assert!( + !key.contains("--") && !key.starts_with('-') && !key.ends_with('-'), + "{key}" + ); +} + +const KEY: &str = "xeon-gold-6142-56c-251g"; + +/// `--costs` first, then `ASAP_CALIBRATION_DIR`, then `/calibration`, +/// then nothing (the formulas). +// provisional answer 15: an `ASAP_CALIBRATION_DIR` without the machine's +// file falls back to the formulas; the repo default is read only when the +// variable is unset. +#[test] +fn path_precedence() { + let env = TempDir::new("env"); + let repo = TempDir::new("repo"); + let given = env.write("given/costs.json", "{}"); + let env_costs = env.write(&format!("{KEY}/costs.json"), "{}"); + let env_accuracy = env.write(&format!("{KEY}/accuracy.json"), "{}"); + let repo_costs = repo.write(&format!("calibration/{KEY}/costs.json"), "{}"); + let overrides = TableOverrides { + costs: Some(given.clone()), + ..TableOverrides::default() + }; + let resolved = resolve_table_paths(&overrides, Some(&env.0), Some(&repo.0), KEY); + assert_eq!(resolved.costs, TablePath::Given(given)); + assert_eq!(resolved.accuracy, TablePath::Calibrated(env_accuracy)); + let resolved = + resolve_table_paths(&TableOverrides::default(), Some(&env.0), Some(&repo.0), KEY); + assert_eq!(resolved.costs, TablePath::Calibrated(env_costs)); + let resolved = resolve_table_paths(&TableOverrides::default(), None, Some(&repo.0), KEY); + assert_eq!(resolved.costs, TablePath::Calibrated(repo_costs)); + assert_eq!( + resolved.accuracy, + TablePath::Missing { + looked_at: repo.0.join(format!("calibration/{KEY}/accuracy.json")) + } + ); + let empty = TempDir::new("empty"); + let resolved = resolve_table_paths( + &TableOverrides::default(), + Some(&empty.0), + Some(&repo.0), + KEY, + ); + assert_eq!( + resolved.costs, + TablePath::Missing { + looked_at: empty.0.join(format!("{KEY}/costs.json")) + } + ); +} + +/// A small workload whose plan the tables can change. +fn rate_workload() -> asap_planner::PlanningWorkload { + workload( + &[("sum by (job) (rate(requests_total[5m]))", exact())], + 10_000, + &[("requests_total", facts(100, 1_000))], + ) +} + +/// No `costs.json` for this machine: one warning naming the machine key +/// and `asap_run calibrate`, and the plan is the default cost inputs' +/// plan, byte for byte. +#[test] +fn missing_costs_warns_and_falls_back() { + let dir = TempDir::new("missing"); + let resolved = resolve_table_paths(&TableOverrides::default(), Some(&dir.0), None, KEY); + let tables = load_planning_tables(&resolved, &TableOverrides::default(), KEY).unwrap(); + let costs_warnings: Vec<_> = tables + .warnings + .iter() + .filter(|warning| warning.contains("costs")) + .collect(); + assert_eq!(costs_warnings.len(), 1, "{:?}", tables.warnings); + assert!(costs_warnings[0].contains(KEY), "{costs_warnings:?}"); + assert!( + costs_warnings[0].contains("asap_run calibrate"), + "{costs_warnings:?}" + ); + let caps = DeploymentCapabilities::default(); + let fallback = plan( + &rate_workload(), + &tables.cost_inputs, + &tables.accuracy, + &caps, + ); + let default = plan( + &rate_workload(), + &CostInputs::default(), + &NoMeasuredGuarantees, + &caps, + ); + assert_eq!( + serde_json::to_string(&fallback.0).unwrap(), + serde_json::to_string(&default.0).unwrap() + ); +} + +fn scan_table() -> MeasuredOperationCosts { + MeasuredOperationCosts { + operators: vec![OperatorCost { + operator: OperatorKind::Scan, + phase: None, + sketch: None, + exact: None, + measure: None, + ns_per_unit: 20.0, + ns_per_invocation: 0.0, + memory_bytes_per_state: None, + }], + ..MeasuredOperationCosts::default() + } +} + +/// Costs present, accuracy absent: exactly one warning, for the accuracy +/// table, and the plan is the costs-only plan with no measured accuracy. +#[test] +fn missing_accuracy_warns_separately() { + let dir = TempDir::new("costs_only"); + dir.write( + &format!("{KEY}/costs.json"), + &serde_json::to_string(&scan_table()).unwrap(), + ); + let resolved = resolve_table_paths(&TableOverrides::default(), Some(&dir.0), None, KEY); + let tables = load_planning_tables(&resolved, &TableOverrides::default(), KEY).unwrap(); + assert_eq!(tables.warnings.len(), 1, "{:?}", tables.warnings); + assert!( + tables.warnings[0].contains("accuracy"), + "{:?}", + tables.warnings + ); + assert_eq!(tables.cost_inputs.measured_operation_costs, scan_table()); + assert!(tables.accuracy.entries.is_empty()); + let costs_only = CostInputs { + measured_operation_costs: scan_table(), + ..CostInputs::default() + }; + let caps = DeploymentCapabilities::default(); + assert_eq!( + plan( + &rate_workload(), + &tables.cost_inputs, + &tables.accuracy, + &caps + ), + plan(&rate_workload(), &costs_only, &NoMeasuredGuarantees, &caps), + ); +} + +/// `--costs` wins over the table inside `--cost-inputs`, with a warning. +// provisional answer 15 +#[test] +fn costs_flag_wins_over_cost_inputs_table() { + let dir = TempDir::new("precedence"); + let inner = MeasuredOperationCosts { + ns_per_evaluation: 1_000.0, + ..scan_table() + }; + let cost_inputs = dir.write( + "cost_inputs.json", + &serde_json::to_string(&CostInputs { + measured_operation_costs: inner, + ..CostInputs::default() + }) + .unwrap(), + ); + let costs = dir.write("costs.json", &serde_json::to_string(&scan_table()).unwrap()); + let overrides = TableOverrides { + costs: Some(costs), + cost_inputs: Some(cost_inputs), + ..TableOverrides::default() + }; + let resolved = resolve_table_paths(&overrides, Some(&dir.0), None, KEY); + let tables = load_planning_tables(&resolved, &overrides, KEY).unwrap(); + assert_eq!(tables.cost_inputs.measured_operation_costs, scan_table()); + assert!( + tables + .warnings + .iter() + .any(|warning| warning.contains("--costs")), + "{:?}", + tables.warnings + ); +} + +/// A truncated or invalid table is an error naming its path, never a +/// silent fallback. +// provisional answer 15 +#[test] +fn corrupt_costs_is_an_error() { + for (name, contents) in [ + ( + "truncated", + r#"{"operators": [{"operator": "Scan", "ns_per_unit""#, + ), + ( + "negative", + r#"{"operators": [{"operator": "Scan", "ns_per_unit": -5.0}]}"#, + ), + ] { + let dir = TempDir::new(name); + let path = dir.write(&format!("{KEY}/costs.json"), contents); + let resolved = resolve_table_paths(&TableOverrides::default(), Some(&dir.0), None, KEY); + let error = load_planning_tables(&resolved, &TableOverrides::default(), KEY).unwrap_err(); + assert_eq!(error.path, path, "{name}"); + } +} + +/// A run of `bin` on `workload_path` with the calibration directory `dir`. +fn run_with_calibration_dir(bin: &str, args: &[&str], dir: &Path) -> String { + let output = Command::new(bin) + .args(args) + .env(CALIBRATION_DIR_ENV, dir) + .output() + .unwrap(); + String::from_utf8_lossy(&output.stderr).into_owned() +} + +/// `asap-planner` and `asap_run` resolve the same tables: with the same +/// empty calibration directory, both print the same missing-table +/// warnings. +#[test] +fn asap_planner_cli_uses_same_resolution() { + let dir = TempDir::new("cli"); + let workload = dir.write("w.json", &serde_json::to_string(&rate_workload()).unwrap()); + let workload = workload.to_str().unwrap(); + let warnings = |stderr: String| -> Vec { + stderr + .lines() + .filter(|line| line.contains("asap_run calibrate")) + .map(str::to_owned) + .collect() + }; + let planner = warnings(run_with_calibration_dir( + env!("CARGO_BIN_EXE_asap_planner"), + &["--workload", workload], + &dir.0, + )); + let run = warnings(run_with_calibration_dir( + env!("CARGO_BIN_EXE_asap_run"), + &[ + "execute", + "--workload", + workload, + "--data", + "facts", + "--at", + "600000", + ], + &dir.0, + )); + assert!( + !planner.is_empty(), + "asap-planner printed no missing-table warning" + ); + assert_eq!(planner, run); +} diff --git a/crates/devtools/tests/measurement/measured_accuracy.rs b/crates/devtools/tests/measurement/measured_accuracy.rs new file mode 100644 index 000000000..bf2d71f8c --- /dev/null +++ b/crates/devtools/tests/measurement/measured_accuracy.rs @@ -0,0 +1,342 @@ +//! §2.4: the accuracy calibration's aggregation, and the measured table +//! read by Pass 2 through the planner. + +use std::collections::BTreeSet; + +use asap_accuracy::{ + ErrorAtFailureProbability, MeasuredAccuracyEntry, MeasuredSketchAccuracy, NoMeasuredGuarantees, +}; +use asap_cost::CostInputs; +use asap_devtools::measurement::accuracy_calibration::{ + error_quantile, worst_over_merge_counts, MergeCountErrors, +}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; +use asap_planner::{DeploymentCapabilities, PlanningModels, PlanningWorkload}; +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +use super::common::{facts, per_value, plan, workload}; + +fn point(failure_probability: f64, error: f64) -> ErrorAtFailureProbability { + ErrorAtFailureProbability { + failure_probability, + error, + } +} + +/// Rank errors of KLL k = 547 at δ = 0.01 after 1, 6 and 360 merges: +/// 0.0015, 0.0018, 0.0017. The table records the worst, 0.0018, not the +/// mean (0.001667) or the last (0.0017). +#[test] +fn worst_over_merge_counts_is_recorded() { + let after = |merge_count, error| MergeCountErrors { + sketch: SketchParams::Kll { k: 547 }, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric: AccuracyMetric::Rank, + merge_count, + trials: 1_000, + error_at: vec![point(0.01, error)], + }; + let entries = + worst_over_merge_counts(&[after(1, 0.0015), after(6, 0.0018), after(360, 0.0017)]); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].error_at, vec![point(0.01, 0.0018)]); + assert_eq!(entries[0].trials, 1_000); +} + +/// Errors i / 1e5 for i = 1..1000, given in descending order: the +/// nearest-rank 0.99 quantile is the ⌈0.99 · 1000⌉ = 990th smallest, +/// 0.0099. +// provisional answer 14 +#[test] +fn quantile_of_errors_nearest_rank() { + let errors: Vec = (1..=1000).rev().map(|i| i as f64 / 1e5).collect(); + assert_eq!(error_quantile(&errors, 0.01), 990.0 / 1e5); +} + +/// `quantile_over_time(0.99, m[5m])` every 10 s over 8 series sampled each +/// second, within rank error 0.0025 per value at δ = 0.01. +fn kll_workload() -> PlanningWorkload { + workload( + &[( + "quantile_over_time(0.99, m[5m])", + per_value("rank", 0.0025, 0.01), + )], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +/// Every KLL `k` Pass 2 connects to a readout through an edge that meets +/// its requirement. +fn admissible_kll_sizes(workload: &PlanningWorkload, models: PlanningModels<'_>) -> BTreeSet { + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_planning_stages_over( + lower_promql_roots(workload).unwrap(), + &entries, + &data, + models, + ) + .unwrap(); + let logical = run.run.logical_asap_candidates.expect("Pass 2 ran"); + logical + .edges + .iter() + .filter(|edge| edge.satisfies_logical_requirement) + .filter(|edge| { + matches!( + logical + .memo + .candidate(edge.consumer_candidate_id) + .expression + .operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + ) + }) + .filter_map(|edge| { + match logical + .memo + .candidate(edge.producer_candidate_id) + .logical_properties + .parameters() + { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + } + }) + .collect() +} + +/// The KLL `k` of the selected plan's sketch, if it has one. +fn selected_kll_size( + workload: &PlanningWorkload, + evidence: &dyn asap_accuracy::AccuracyEvidence, +) -> Option { + let (installable, _) = plan( + workload, + &CostInputs::default(), + evidence, + &DeploymentCapabilities::default(), + ); + installable + .dag + .nodes + .iter() + .find_map(|node| match node.parameters.as_ref() { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + }) +} + +/// Without a table, the closed form sizes KLL: of the predefined sizes, +/// k = 547 (rank error 0.004998 at δ = 0.01) misses 0.0025 and the smallest +/// admissible is k = 1404 (0.00200). +#[test] +fn kll_sized_by_closed_form_without_measured_table() { + let sizes = admissible_kll_sizes(&kll_workload(), PlanningModels::builtin()); + assert!(!sizes.contains(&547), "{sizes:?}"); + assert_eq!(sizes.first(), Some(&1_404), "{sizes:?}"); +} + +/// The fixture table: KLL k = 547 for the 0.99 quantile measured at rank +/// error 0.0024 at δ = 0.01 (within 0.0025). +fn kll_547_measured(metric: AccuracyMetric, k: u32) -> MeasuredAccuracyEntry { + MeasuredAccuracyEntry { + sketch: SketchParams::Kll { k }, + statistic: SketchStatistic::Quantile { q: 0.99 }, + accuracy_metric: metric, + trials: 1_000, + error_at: vec![ + point(0.003, 0.0031), + point(0.01, 0.0024), + point(0.05, 0.0016), + ], + } +} + +/// With the table, k = 547 is admissible and, its state being smaller than +/// k = 1404's, selected. +#[test] +fn measured_accuracy_shrinks_sketch() { + let table = MeasuredSketchAccuracy { + entries: vec![kll_547_measured(AccuracyMetric::Rank, 547)], + }; + let sizes = admissible_kll_sizes(&kll_workload(), PlanningModels::new(&table)); + assert!(sizes.contains(&547), "{sizes:?}"); + assert_eq!(selected_kll_size(&kll_workload(), &table), Some(547)); +} + +/// Entries for k = 547 and k = 1404 in the relative metric (not KLL's +/// rank) neither admit k = 547 nor prune k = 1404: the plan is the plan +/// without a table. +// provisional answer 7 +#[test] +fn wrong_metric_entry_does_not_prune() { + let table = MeasuredSketchAccuracy { + entries: vec![ + kll_547_measured(AccuracyMetric::Relative, 547), + kll_547_measured(AccuracyMetric::Relative, 1_404), + ], + }; + let with = admissible_kll_sizes(&kll_workload(), PlanningModels::new(&table)); + let without = admissible_kll_sizes(&kll_workload(), PlanningModels::builtin()); + assert_eq!(with, without); + assert!(with.contains(&1_404), "{with:?}"); + let caps = DeploymentCapabilities::default(); + assert_eq!( + plan(&kll_workload(), &CostInputs::default(), &table, &caps), + plan( + &kll_workload(), + &CostInputs::default(), + &NoMeasuredGuarantees, + &caps + ), + ); +} + +/// An empty table plans exactly as no measurement: the same plan, cost +/// report and pruned candidates (I7). +#[test] +fn empty_table_equals_no_measured() { + let workload = kll_workload(); + let empty = MeasuredSketchAccuracy::default(); + let caps = DeploymentCapabilities::default(); + assert_eq!( + plan(&workload, &CostInputs::default(), &empty, &caps), + plan( + &workload, + &CostInputs::default(), + &NoMeasuredGuarantees, + &caps + ), + ); + let pruned = |models: PlanningModels<'_>| { + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_planning_stages_over( + lower_promql_roots(&workload).unwrap(), + &entries, + &data, + models, + ) + .unwrap(); + format!( + "{:?} {:?}", + run.run + .logical_asap_candidates + .map(|logical| logical.pruned), + run.run + .physical_asap_candidates + .map(|physical| physical.pruned_candidates) + ) + }; + assert_eq!( + pruned(PlanningModels::new(&empty)), + pruned(PlanningModels::builtin()) + ); +} + +/// A small calibration's accuracy table holds I4: per entry, δ ascending +/// in (0, 1] and ≥ 3 / trials, errors finite, non-negative and +/// non-increasing in δ, and each the worst of the per-merge-count results +/// at that δ. +// provisional answer 13 +#[test] +fn accuracy_table_is_monotone() { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: true, + seed: 7, + }) + .unwrap(); + calibration.accuracy.validate().unwrap(); + for entry in &calibration.accuracy.entries { + let floor = 3.0 / entry.trials as f64; + for pair in entry.error_at.windows(2) { + assert!( + pair[0].failure_probability < pair[1].failure_probability, + "{entry:?}" + ); + assert!(pair[1].error <= pair[0].error, "{entry:?}"); + } + for at in &entry.error_at { + assert!( + at.failure_probability >= floor && at.failure_probability <= 1.0, + "{entry:?}" + ); + assert!(at.error.is_finite() && at.error >= 0.0, "{entry:?}"); + let worst = calibration + .accuracy_by_merge_count + .iter() + .filter(|raw| { + raw.sketch == entry.sketch + && raw.statistic == entry.statistic + && raw.accuracy_metric == entry.accuracy_metric + }) + .flat_map(|raw| raw.error_at.iter()) + .filter(|raw| raw.failure_probability == at.failure_probability) + .map(|raw| raw.error) + .fold(f64::NEG_INFINITY, f64::max); + assert_eq!(at.error, worst, "{entry:?}"); + } + } +} + +/// A full calibration's measurements lie within the sketches' known +/// bounds: DDSketch α = 0.01 never exceeds relative error 0.01 (a +/// deterministic bound); CMS (w, d) at δ ≥ e^−d plus a 99% binomial slack +/// has absolute error at most e / w of ‖f‖₁. KLL k = 547 at δ = 0.01 +/// within 1.2 × 0.004998 is a sanity check that only warns. +#[test] +#[ignore = "timing: run on a dedicated host"] +fn measured_within_known_bounds() { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: false, + seed: 7, + }) + .unwrap(); + for raw in &calibration.accuracy_by_merge_count { + match (&raw.sketch, raw.accuracy_metric) { + (SketchParams::DDSketch { alpha }, AccuracyMetric::Relative) if *alpha == 0.01 => { + for at in &raw.error_at { + assert!(at.error <= 0.01 * (1.0 + 1e-9), "{raw:?}"); + } + } + (SketchParams::Cms { width, depth }, metric) + if metric == AccuracyMetric::ABSOLUTE_STREAM_L1 => + { + let trials = raw.trials as f64; + let failure = (-(*depth as f64)).exp(); + // One-sided 99% slack on a binomial fraction of `trials`. + let slack = 2.326 * (failure * (1.0 - failure) / trials).sqrt(); + for at in raw + .error_at + .iter() + .filter(|at| at.failure_probability >= failure + slack) + { + assert!(at.error <= std::f64::consts::E / *width as f64, "{raw:?}"); + } + } + (SketchParams::Kll { k: 547 }, AccuracyMetric::Rank) => { + for at in raw + .error_at + .iter() + .filter(|at| at.failure_probability == 0.01) + { + if at.error > 1.2 * 0.004998 { + eprintln!( + "warning: KLL k = 547 measured {} > 1.2 x 0.004998", + at.error + ); + } + } + } + _ => {} + } + } +} diff --git a/crates/devtools/tests/measurement/mod.rs b/crates/devtools/tests/measurement/mod.rs new file mode 100644 index 000000000..30f4763a1 --- /dev/null +++ b/crates/devtools/tests/measurement/mod.rs @@ -0,0 +1,20 @@ +//! The `asap_run` test suite (`asap_devtools::measurement`), by the test +//! plan's sections. Tiers: T0 and T1 run in CI; T2 (timing) is ignored +//! with "timing: run on a dedicated host"; T3 (milestone scale) with +//! "milestone scale". Every expected value is computed by hand, from a +//! kernel or clock of known cost, from known sketch bounds or by +//! independent enumeration, never by the planner's own optimization or +//! cost code. A test that depends on a provisional answer of the test plan +//! says `// provisional answer N`. + +mod common; +mod cost_accounting; +mod cost_fit; +mod end_to_end; +mod execution; +mod gate; +mod machine_and_tables; +mod measured_accuracy; +mod oracle; +mod regression; +mod source_facts; diff --git a/crates/devtools/tests/measurement/oracle.rs b/crates/devtools/tests/measurement/oracle.rs new file mode 100644 index 000000000..6a07af362 --- /dev/null +++ b/crates/devtools/tests/measurement/oracle.rs @@ -0,0 +1,337 @@ +//! §2.11: the exhaustive oracle. Every plan over Stage 2's candidates is +//! enumerated independently of Stage 3 +//! (`asap_types::test_support::physical_plan_enumeration`), each is +//! measured, and the selected plan must be within τ of the cheapest +//! measured one, τ = 0.05 + twice the cheapest plan's coefficient of +//! variation over its repetitions. +// provisional answer 17 + +use std::collections::BTreeSet; + +use asap_devtools::measurement::benchmark::{ + bench_plan, installable_plan_of_selection, ArmMeasurement, BenchArm, BenchRequest, DataSetting, +}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest, PlanningTables}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_devtools::planning_stages::{ + lower_promql_roots, run_planning_stages_over, PlanningStagesRun, +}; +use asap_planner::{PlanningModels, PlanningWorkload}; +use asap_types::stages::PhysicalCandidateId; +use asap_types::test_support::physical_plan_enumeration::{ + every_physical_plan, PhysicalPlanChoice, +}; + +use super::common::{exact, facts, per_value, plan, workload}; + +const MOST_PLANS: usize = 1_000_000; + +fn exact_family() -> PlanningWorkload { + workload( + &[("sum by (job) (sum_over_time(m[5m]))", exact())], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +fn sketch_family() -> PlanningWorkload { + workload( + &[( + "quantile_over_time(0.99, m[5m])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +fn joint_sharing() -> PlanningWorkload { + workload( + &[ + ("sum by (job) (sum_over_time(m[5m]))", exact()), + ( + "quantile_over_time(0.99, m[5m])", + per_value("relative", 0.01, 0.01), + ), + ("count_over_time(m[5m])", exact()), + ], + 10_000, + &[("m", facts(8, 1_000))], + ) +} + +fn planning_run(workload: &PlanningWorkload, models: PlanningModels<'_>) -> PlanningStagesRun { + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + run_planning_stages_over( + lower_promql_roots(workload).unwrap(), + &entries, + &data, + models, + ) + .unwrap() +} + +/// Every enumerated plan is closed: each query root's group has a picked +/// candidate, one candidate per group, and each picked candidate's every +/// input has one admissible edge whose consumer is that candidate and +/// whose producer is the candidate picked for the producer's group. +#[test] +fn enumerated_plans_are_closed_under_edges() { + for workload in [exact_family(), sketch_family()] { + let run = planning_run(&workload, PlanningModels::builtin()); + let physical = run.run.physical_asap_candidates.expect("Stage 2 ran"); + let plans = every_physical_plan(&physical, MOST_PLANS).unwrap(); + assert!(!plans.is_empty()); + let distinct: BTreeSet<_> = plans.iter().collect(); + assert_eq!(distinct.len(), plans.len(), "a plan is enumerated twice"); + for plan in &plans { + for query in physical.logical_candidates.memo.queries() { + for group in query.root.operator_refs() { + assert!(plan.candidate_of_group.contains_key(group), "{plan:?}"); + } + } + for (&group, &candidate) in &plan.candidate_of_group { + assert_eq!( + physical.physical_candidates[candidate.0] + .logical_candidate_id + .group, + group + ); + for input in 0..physical.input_count(candidate) { + let edge_id = plan.edge_into_input[&(candidate, input)]; + assert!(physical + .admissible_edges_into(candidate, input) + .contains(&edge_id)); + let edge = physical.edge(edge_id); + assert_eq!(edge.consumer_candidate_id, candidate); + let producer_group = physical.physical_candidates[edge.producer_candidate_id.0] + .logical_candidate_id + .group; + assert_eq!( + plan.candidate_of_group[&producer_group], + edge.producer_candidate_id + ); + } + } + assert_eq!( + plan.edge_into_input.len(), + plan.candidate_of_group + .values() + .map(|&candidate| physical.input_count(candidate)) + .sum::() + ); + } + } +} + +/// The plan Stage 3 selects is one of the enumerated plans (validity): +/// its nodes' physical candidates are exactly one plan's candidates. +#[test] +fn enumerated_plans_contain_the_selected_plan() { + for workload in [exact_family(), sketch_family()] { + let run = planning_run(&workload, PlanningModels::builtin()); + let (dag, _) = run.run.selection.as_ref().expect("a plan is selected"); + let selected: BTreeSet = dag + .nodes + .iter() + .filter_map(|node| node.selected_physical_candidate) + .collect(); + let physical = run + .run + .physical_asap_candidates + .as_ref() + .expect("Stage 2 ran"); + let plans = every_physical_plan(physical, MOST_PLANS).unwrap(); + assert!( + plans.iter().any(|plan| plan.candidates() == selected), + "selected {selected:?} is none of the {} enumerated plans", + plans.len() + ); + } +} + +/// The joint workload's plans are few enough to measure each: at most +/// [`MOST_PLANS`]. Stage 2 gives it 452 physical candidates in 23 groups +/// (the exact family has 27 and 624 plans). +#[test] +fn joint_sharing_plan_space_is_enumerable() { + let run = planning_run(&joint_sharing(), PlanningModels::builtin()); + let physical = run.run.physical_asap_candidates.expect("Stage 2 ran"); + let plans = every_physical_plan(&physical, MOST_PLANS); + assert!(plans.is_ok(), "{plans:?}"); +} + +/// The calibrated tables of this machine. +fn calibrated_tables() -> PlanningTables { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: false, + seed: 7, + }) + .unwrap(); + PlanningTables { + cost_inputs: asap_cost::CostInputs { + measured_operation_costs: calibration.costs, + ..asap_cost::CostInputs::default() + }, + accuracy: calibration.accuracy, + warnings: Vec::new(), + } +} + +/// Every enumerated plan and the selected one, measured. +struct OracleRun { + enumerated: Vec<(PhysicalPlanChoice, ArmMeasurement)>, + selected: ArmMeasurement, +} + +fn run_oracle(workload: &PlanningWorkload) -> OracleRun { + let tables = calibrated_tables(); + let models = PlanningModels::new(&tables.accuracy).with_cost_inputs(&tables.cost_inputs); + let run = planning_run(workload, models); + let physical = run.run.physical_asap_candidates.expect("Stage 2 ran"); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let setting = DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms: 10_000, + raw_data_retained: true, + }; + let request = BenchRequest { + workload, + arms: vec![BenchArm::Planned], + settings: vec![setting.clone()], + series_count: 8, + duration_ms: 1_200_000, + repetitions: 5, + tables: tables.clone(), + seed: 1, + cost_gate: true, + output_root: None, + }; + let enumerated = every_physical_plan(&physical, MOST_PLANS) + .unwrap() + .into_iter() + .map(|choice| { + let installable = installable_plan_of_selection( + &physical, + &entries, + &choice.candidates(), + &choice.edges(), + ) + .unwrap(); + let measured = bench_plan(&installable, workload, &setting, &request).unwrap(); + (choice, measured) + }) + .collect(); + let (selected_plan, _) = plan( + workload, + &tables.cost_inputs, + &tables.accuracy, + &asap_planner::DeploymentCapabilities::default(), + ); + let selected = bench_plan(&selected_plan, workload, &setting, &request).unwrap(); + OracleRun { + enumerated, + selected, + } +} + +fn measured_cost(arm: &ArmMeasurement) -> f64 { + arm.measured_cost_per_second.expect("a measured cost") +} + +/// The selected plan's measured cost is within τ of the cheapest +/// enumerated plan's. +// provisional answer 23 +fn assert_within_tau(oracle: &OracleRun) { + let cheapest = oracle + .enumerated + .iter() + .map(|(_, arm)| arm) + .min_by(|a, b| measured_cost(a).total_cmp(&measured_cost(b))) + .expect("at least one plan"); + let repetitions = &cheapest.repetition_costs_per_second; + let mean = repetitions.iter().sum::() / repetitions.len() as f64; + let variance = repetitions + .iter() + .map(|cost| (cost - mean).powi(2)) + .sum::() + / repetitions.len() as f64; + let tau = 0.05 + 2.0 * variance.sqrt() / mean; + assert!( + measured_cost(&oracle.selected) <= (1.0 + tau) * measured_cost(cheapest), + "selected {} vs cheapest {} (tau {tau})", + measured_cost(&oracle.selected), + measured_cost(cheapest) + ); +} + +/// Spearman's rank correlation of two equally long samples, no ties +/// assumed. +fn spearman(left: &[f64], right: &[f64]) -> f64 { + let ranks = |values: &[f64]| { + let mut order: Vec = (0..values.len()).collect(); + order.sort_by(|&a, &b| values[a].total_cmp(&values[b])); + let mut ranks = vec![0.0; values.len()]; + for (rank, index) in order.into_iter().enumerate() { + ranks[index] = rank as f64; + } + ranks + }; + let (left, right) = (ranks(left), ranks(right)); + let n = left.len() as f64; + let d2: f64 = left.iter().zip(&right).map(|(a, b)| (a - b).powi(2)).sum(); + 1.0 - 6.0 * d2 / (n * (n * n - 1.0)) +} + +/// The exact family (exact from raw data, a query-time `SummaryAgg`, a +/// chain at each feasible width, no-raw variants): the selected plan is +/// within τ, and predicted and measured costs rank the plans alike +/// (Spearman ρ ≥ 0.8). +#[test] +#[ignore = "timing: run on a dedicated host"] +fn oracle_exact_family() { + let oracle = run_oracle(&exact_family()); + assert_within_tau(&oracle); + let predicted: Vec = oracle + .enumerated + .iter() + .map(|(_, arm)| arm.predicted.cost_per_second.unwrap()) + .collect(); + let measured: Vec = oracle + .enumerated + .iter() + .map(|(_, arm)| measured_cost(arm)) + .collect(); + assert!(spearman(&predicted, &measured) >= 0.8); +} + +/// The sketch family (exact, DDSketch α ≤ 0.01 at query time or as a +/// chain; KLL excluded by the relative metric): every enumerated plan meets +/// the target, and the selected one is within τ. +// provisional answer 3 +#[test] +#[ignore = "timing: run on a dedicated host"] +fn oracle_sketch_family() { + let oracle = run_oracle(&sketch_family()); + for (choice, arm) in &oracle.enumerated { + assert!( + arm.measured_error.unwrap() <= 0.01 * (1.0 + 1e-9), + "{choice:?}" + ); + } + assert_within_tau(&oracle); +} + +/// The two families with `count_over_time(m[5m])`: the selected shared +/// plan is within τ, and the shared node is predicted once. +#[test] +#[ignore = "timing: run on a dedicated host"] +fn oracle_joint_sharing() { + let oracle = run_oracle(&joint_sharing()); + assert_within_tau(&oracle); + let nodes = &oracle.selected.predicted.predicted_usage.per_node; + let ids: BTreeSet<_> = nodes.iter().map(|node| node.node_id).collect(); + assert_eq!(ids.len(), nodes.len(), "a node is predicted twice"); +} diff --git a/crates/devtools/tests/measurement/regression.rs b/crates/devtools/tests/measurement/regression.rs new file mode 100644 index 000000000..c4bf33b4f --- /dev/null +++ b/crates/devtools/tests/measurement/regression.rs @@ -0,0 +1,186 @@ +//! §2.10: regression tests for the mis-rankings the feasibility run found, +//! with tables calibrated on the same machine. The small variants (100 +//! series, 70 minutes of data so a 1h window is covered) are T2; the full +//! ones (1,000 series) are T3. + +use asap_devtools::measurement::benchmark::{ + bench, bench_showcase, ArmMeasurement, BenchArm, BenchReport, BenchRequest, DataSetting, +}; +use asap_devtools::measurement::calibration_tables::{calibrate, CalibrateRequest, PlanningTables}; +use asap_devtools::measurement::cost_calibration::CalibrationFamilies; +use asap_planner::PlanningWorkload; +use asap_types::ir::properties::ExecutionTiming; + +use super::common::{exact, facts, per_value, workload}; + +fn calibrated_tables() -> PlanningTables { + let calibration = calibrate(&CalibrateRequest { + families: CalibrationFamilies::All, + small: false, + seed: 7, + }) + .unwrap(); + PlanningTables { + cost_inputs: asap_cost::CostInputs { + measured_operation_costs: calibration.costs, + ..asap_cost::CostInputs::default() + }, + accuracy: calibration.accuracy, + warnings: Vec::new(), + } +} + +fn arm(report: &BenchReport, which: BenchArm) -> &ArmMeasurement { + report + .arms + .iter() + .find(|arm| arm.arm == which) + .expect("the arm ran") +} + +fn cost(arm: &ArmMeasurement) -> f64 { + arm.measured_cost_per_second.expect("a measured cost") +} + +fn run(workload: &PlanningWorkload, series: u64, evaluation_interval_ms: u64) -> BenchReport { + bench(&BenchRequest { + workload, + arms: vec![BenchArm::Exact, BenchArm::Planned, BenchArm::NoRaw], + settings: vec![DataSetting { + sample_interval_ms: 1_000, + evaluation_interval_ms, + raw_data_retained: true, + }], + series_count: series, + duration_ms: 70 * 60_000, + repetitions: 5, + tables: calibrated_tables(), + seed: 1, + cost_gate: true, + output_root: None, + }) + .unwrap() +} + +/// `sum by (job) (rate(requests_total[1h]))` at 1 Hz every 300 s with raw +/// data: today a query-time `SummaryAgg` (331 ms p50) is selected though +/// the stored chain is 45× faster, and predicted wrongly ranked. The +/// selected plan must cost at most 1.05 × the cheaper of exact and the +/// chain, and the predicted order of the two must be the measured one. +// provisional answer 1 and provisional answer 20 +fn rate_every_300s(series: u64) { + let workload = workload( + &[("sum by (job) (rate(requests_total[1h]))", exact())], + 300_000, + &[("requests_total", facts(series, 1_000))], + ); + let report = run(&workload, series, 300_000); + let (exact_arm, planned, chain) = ( + arm(&report, BenchArm::Exact), + arm(&report, BenchArm::Planned), + arm(&report, BenchArm::NoRaw), + ); + assert!(cost(planned) <= 1.05 * cost(exact_arm).min(cost(chain))); + let predicted = |arm: &ArmMeasurement| arm.predicted.cost_per_second.unwrap(); + assert_eq!( + predicted(exact_arm) < predicted(chain), + cost(exact_arm) < cost(chain), + "predicted and measured disagree on query time against the chain" + ); +} + +#[test] +#[ignore = "timing: run on a dedicated host"] +fn regression_rate_1hz_every_300s() { + rate_every_300s(100); +} + +#[test] +#[ignore = "milestone scale"] +fn regression_rate_1hz_every_300s_full() { + rate_every_300s(1_000); +} + +/// `quantile_over_time(0.99, latency[1h])` at 1 Hz every 10 s, 1% relative +/// at δ = 0.01, raw data: today a DDSketch built from raw data at query +/// time runs at 0.50× exact. The selected plan must cost at most 1.05 × +/// exact (speedup ≥ 0.95×). +// provisional answer 1 +fn quantile_not_slower(series: u64) { + let workload = workload( + &[( + "quantile_over_time(0.99, latency[1h])", + per_value("relative", 0.01, 0.01), + )], + 10_000, + &[("latency", facts(series, 1_000))], + ); + let report = run(&workload, series, 10_000); + let (exact_arm, planned) = ( + arm(&report, BenchArm::Exact), + arm(&report, BenchArm::Planned), + ); + assert!(cost(planned) <= 1.05 * cost(exact_arm)); + let speedup = exact_arm.measured.as_ref().unwrap().p50_query_ms + / planned.measured.as_ref().unwrap().p50_query_ms; + assert!(speedup >= 0.95, "speedup {speedup}"); +} + +#[test] +#[ignore = "timing: run on a dedicated host"] +fn regression_quantile_half_speed() { + quantile_not_slower(100); +} + +#[test] +#[ignore = "milestone scale"] +fn regression_quantile_half_speed_full() { + quantile_not_slower(1_000); +} + +/// Over the full showcase, the planner ranks the selected plan against +/// exact as measured in at least 90% of the rows, and measured / predicted +/// stays within 1.5× per plan kind (by the timing of the plan's first +/// root). +#[test] +#[ignore = "milestone scale"] +fn regression_ranking_rate() { + let report = bench_showcase(false, &calibrated_tables(), None).unwrap(); + let mut right = 0; + let mut rows = 0; + for exact_arm in report.arms.iter().filter(|arm| arm.arm == BenchArm::Exact) { + let Some(planned) = report.arms.iter().find(|arm| { + arm.arm == BenchArm::Planned + && arm.setting == exact_arm.setting + && arm.plan.query_texts == exact_arm.plan.query_texts + }) else { + continue; + }; + rows += 1; + let predicted_order = + planned.predicted.cost_per_second < exact_arm.predicted.cost_per_second; + let measured_order = cost(planned) < cost(exact_arm); + right += usize::from(predicted_order == measured_order); + } + assert!(rows > 0); + assert!( + right as f64 >= 0.9 * rows as f64, + "{right} of {rows} ranked right" + ); + for kind in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] { + for arm in report.arms.iter().filter(|arm| { + let root = arm.plan.dag.roots[0]; + arm.plan + .dag + .nodes + .iter() + .any(|node| node.id == root && node.output_state.timing == kind) + }) { + let ratio = cost(arm) / arm.predicted.cost_per_second.unwrap(); + assert!( + (1.0 / 1.5..=1.5).contains(&ratio), + "{kind:?}: measured / predicted {ratio}" + ); + } + } +} diff --git a/crates/devtools/tests/measurement/source_facts.rs b/crates/devtools/tests/measurement/source_facts.rs new file mode 100644 index 000000000..2e03d1ef7 --- /dev/null +++ b/crates/devtools/tests/measurement/source_facts.rs @@ -0,0 +1,447 @@ +//! §2.6: source facts from hand-made data. +//! +//! Source `m`: +//! +//! | Series | Samples (t ms : value) | +//! |---|---| +//! | A `{job=a,instance=1}` | 0:1, 1000:2, 2000:−3, 4000:0 | +//! | B `{job=a,instance=2}` | 0:5, 1000:10 | +//! | C `{job=b,instance=1}` | 500:0.5, 1500:0.25, 2500:100 | +//! | D `{job=b}` (no instance) | 0:7, 2000:7 | +//! +//! Source `lat`: two series, every value in [0.001, 3], one of them −0.0. + +use aqpbm_datagen::{DataDistribution, SeriesLabelSpec, TimeSeriesDescription, UniformParameter}; +use asap_devtools::measurement::samples::{read_samples_jsonl, samples_of_generated, Sample}; +use asap_devtools::measurement::source_facts::{ + observe_source_statistics, source_facts_of_samples, FactsOptions, ObservedMagnitude, + SuggestedFact, +}; +use asap_planner::PlanningWorkload; +use asap_types::stages::ValueSign; +use asap_types::workload::LabelSetDistinctValues; + +use super::common::{per_value, sample, whole_result, workload}; + +fn fixture() -> Vec { + let a = [("job", "a"), ("instance", "1")]; + let b = [("job", "a"), ("instance", "2")]; + let c = [("job", "b"), ("instance", "1")]; + let d = [("job", "b")]; + let mut samples = vec![ + sample("m", &a, 0, 1.0), + sample("m", &a, 1_000, 2.0), + sample("m", &a, 2_000, -3.0), + sample("m", &a, 4_000, 0.0), + sample("m", &b, 0, 5.0), + sample("m", &b, 1_000, 10.0), + sample("m", &c, 500, 0.5), + sample("m", &c, 1_500, 0.25), + sample("m", &c, 2_500, 100.0), + sample("m", &d, 0, 7.0), + sample("m", &d, 2_000, 7.0), + ]; + let x = [("host", "x")]; + let y = [("host", "y")]; + samples.extend([ + sample("lat", &x, 0, 0.001), + sample("lat", &x, 1_000, 3.0), + sample("lat", &y, 0, -0.0), + sample("lat", &y, 1_000, 1.5), + ]); + samples +} + +const QUERY: &str = "sum by (job, instance) (sum_over_time(m[2s]))"; +const LAT_QUERY: &str = "max_over_time(lat[2s])"; + +/// The fixture's workload, with per-value targets: no fact splits δ, so +/// the label counts are cost-only. +fn per_value_workload() -> PlanningWorkload { + workload( + &[ + (QUERY, per_value("absolute", 1.0, 0.01)), + (LAT_QUERY, per_value("absolute", 1.0, 0.01)), + ], + 1_000, + &[], + ) +} + +/// The same with a whole-result target: the series and label counts bound +/// G, over which δ splits, so they become guarantee facts. +fn whole_result_workload() -> PlanningWorkload { + workload( + &[ + (QUERY, whole_result("absolute", 1.0, 0.01)), + (LAT_QUERY, per_value("absolute", 1.0, 0.01)), + ], + 1_000, + &[], + ) +} + +fn observed_m() -> asap_devtools::measurement::source_facts::ObservedSourceStatistics { + observe_source_statistics(&fixture(), &per_value_workload()) + .remove("m") + .unwrap() +} + +/// Four distinct label sets. +#[test] +fn facts_series_count() { + assert_eq!(observed_m().series_count, 4); +} + +/// Gaps A 1000, 1000, 2000; B 1000; C 1000, 1000; D 2000. Per-series +/// medians 1000, 1000, 1000, 2000, whose median is 1000 (the mean gap, +/// 1286, is wrong). A second fixture separates the two medians: P has 10 +/// samples 1 s apart, Q and R 2 samples 5 s apart; the pooled median gap +/// is 1000, the median of per-series medians 5000. +// provisional answer 19 +#[test] +fn facts_sample_interval_median_gap() { + assert_eq!(observed_m().sample_interval_ms, Some(1_000)); + let mut samples: Vec = (0..10) + .map(|i| sample("s", &[("id", "p")], i * 1_000, 1.0)) + .collect(); + for id in ["q", "r"] { + samples.push(sample("s", &[("id", id)], 0, 1.0)); + samples.push(sample("s", &[("id", id)], 5_000, 1.0)); + } + let workload = workload( + &[("max_over_time(s[10s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&samples, &workload) + .remove("s") + .unwrap(); + assert_eq!(observed.sample_interval_ms, Some(5_000)); +} + +/// `job` takes 2 values; `instance` 3 (1, 2 and missing on D). +#[test] +fn facts_distinct_per_label() { + let observed = observed_m(); + assert_eq!(observed.distinct_values_per_label["job"], 2); + assert_eq!(observed.distinct_values_per_label["instance"], 3); +} + +/// `by (job, instance)` sees 4 combinations (a/1, a/2, b/1, b/missing), +/// not the product 6; `lat`'s query groups by no label set of two, so it +/// gets no entry. +// provisional answer 19 +#[test] +fn facts_label_set_from_workload_by() { + let observed = observe_source_statistics(&fixture(), &per_value_workload()); + let mut sets = observed["m"].distinct_values_per_label_set.clone(); + for set in &mut sets { + set.labels.sort(); + } + assert_eq!( + sets, + vec![LabelSetDistinctValues { + labels: vec!["instance".into(), "job".into()], + distinct_values: 4, + }] + ); + assert!(observed["lat"].distinct_values_per_label_set.is_empty()); +} + +/// Zeros are excluded: `m.value` spans 0.25 to 100 (a min_abs of 0 would +/// be invalid). +#[test] +fn facts_column_magnitudes_exclude_zero() { + assert_eq!( + observed_m().column_magnitudes["value"], + ObservedMagnitude { + min_abs: 0.25, + max_abs: 100.0, + } + ); + let zeros = vec![sample("z", &[], 0, 0.0), sample("z", &[], 1_000, -0.0)]; + let workload = workload( + &[("max_over_time(z[2s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&zeros, &workload) + .remove("z") + .unwrap(); + assert!(observed.column_magnitudes.get("value").is_none()); +} + +/// Signs by sign bit: `m` has −3, so no sign suggestion; `lat` holds +/// −0.0, so no `non_negative` either (a `v >= 0.0` check would accept it). +#[test] +fn facts_sign_suggestion_sign_bit() { + let report = + source_facts_of_samples(&fixture(), &per_value_workload(), FactsOptions::default()) + .unwrap(); + assert_eq!( + report.observed["m"].column_signs["value"], + ValueSign::Unknown + ); + assert_ne!( + report.observed["lat"].column_signs["value"], + ValueSign::NonNegative + ); + assert!( + !report.suggestions.iter().any(|suggestion| matches!( + &suggestion.fact, + SuggestedFact::ColumnSign { + sign: ValueSign::NonNegative, + .. + } + )), + "{:?}", + report.suggestions + ); +} + +/// With per-value targets, the cost-only facts are written as observed: +/// series count 4, interval 1000, label counts 2 and 3, the label set's 4; +/// the magnitudes with their margin (0.125 to 200). +// provisional answer 5 and provisional answer 6 +#[test] +fn facts_cost_only_written_directly() { + let report = + source_facts_of_samples(&fixture(), &per_value_workload(), FactsOptions::default()) + .unwrap(); + let m = &report.data_workload.sources["m"]; + assert_eq!(m.series_count, Some(4)); + assert_eq!(m.sample_interval.map(|interval| interval.0), Some(1_000)); + assert_eq!(m.distinct_values_per_label["job"], 2); + assert_eq!(m.distinct_values_per_label["instance"], 3); + assert_eq!(m.distinct_values_per_label_set[0].distinct_values, 4); + assert_eq!( + report.column_magnitudes["m"]["value"], + ObservedMagnitude { + min_abs: 0.125, + max_abs: 200.0, + } + ); +} + +/// With a whole-result target, the guarantee facts are suggested with a +/// ×2 margin, ⌈2×⌉ for counts: instance 6, job 4, series 8, the label set +/// 8; distinct values per 2 s window at most 2 observed, so 4; norms over +/// one group's 2 s window at most ‖f‖₁ = 2 and ‖f‖₂ = √2 observed, so 4 +/// and 2√2. They are written only with `--trust-observed-bounds`. +// provisional answer 5 and provisional answer 19 +#[test] +fn facts_guarantee_margin() { + let suggested = source_facts_of_samples( + &fixture(), + &whole_result_workload(), + FactsOptions::default(), + ) + .unwrap(); + let observed = &suggested.observed["m"]; + assert_eq!(observed.max_distinct_per_evaluation, Some(2)); + let norms = observed.stream_norms.unwrap(); + assert_eq!(norms.l1, Some(2.0)); + assert!((norms.l2.unwrap() - 2f64.sqrt()).abs() < 1e-12); + let m_facts: Vec<&SuggestedFact> = suggested + .suggestions + .iter() + .filter(|suggestion| suggestion.source == "m") + .map(|suggestion| &suggestion.fact) + .collect(); + let has = |fact: &SuggestedFact| m_facts.contains(&fact); + assert!( + has(&SuggestedFact::DistinctValuesPerLabel { + label: "instance".into(), + distinct_values: 6 + }), + "{m_facts:?}" + ); + assert!( + has(&SuggestedFact::DistinctValuesPerLabel { + label: "job".into(), + distinct_values: 4 + }), + "{m_facts:?}" + ); + assert!(has(&SuggestedFact::SeriesCount(8)), "{m_facts:?}"); + assert!( + has(&SuggestedFact::MaxDistinctPerEvaluation(4)), + "{m_facts:?}" + ); + assert!( + m_facts + .iter() + .any(|fact| matches!(fact, SuggestedFact::StreamNorms(norms) + if norms.l1 == Some(4.0) && (norms.l2.unwrap() - 2.0 * 2f64.sqrt()).abs() < 1e-12)), + "{m_facts:?}" + ); + // Not written by default. + let m = &suggested.data_workload.sources["m"]; + assert_eq!(m.series_count, None); + assert!(m.distinct_values_per_label.is_empty()); + assert_eq!(m.max_distinct_per_evaluation, None); + assert_eq!(m.stream_norms, None); + // Written when trusted. + let trusted = source_facts_of_samples( + &fixture(), + &whole_result_workload(), + FactsOptions { + trust_observed_bounds: true, + }, + ) + .unwrap(); + let m = &trusted.data_workload.sources["m"]; + assert_eq!(m.series_count, Some(8)); + assert_eq!(m.distinct_values_per_label["instance"], 6); + assert_eq!(m.max_distinct_per_evaluation, Some(4)); +} + +/// After the margins, the label set's count (⌈2 · 4⌉ = 8) stays at most +/// the product of its labels' counts (4 · 6 = 24), and the written +/// workload validates. +#[test] +fn facts_margined_set_stays_valid() { + let trusted = source_facts_of_samples( + &fixture(), + &whole_result_workload(), + FactsOptions { + trust_observed_bounds: true, + }, + ) + .unwrap(); + let m = &trusted.data_workload.sources["m"]; + for set in &m.distinct_values_per_label_set { + let product: u64 = set + .labels + .iter() + .map(|label| m.distinct_values_per_label[label]) + .product(); + assert!(set.distinct_values <= product, "{set:?}"); + } + let mut written = whole_result_workload(); + written.data_workload = Some(trusted.data_workload.clone()); + written.validate().unwrap(); +} + +/// A counter 0, 5, 10, 2, 7 resets once: still non-negative; magnitudes +/// over the raw samples (zeros excluded): 2 to 10. +#[test] +fn facts_counter_reset() { + let samples: Vec = [0.0, 5.0, 10.0, 2.0, 7.0] + .iter() + .enumerate() + .map(|(i, &value)| sample("c_total", &[], i as i64 * 1_000, value)) + .collect(); + let workload = workload( + &[("rate(c_total[5s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&samples, &workload) + .remove("c_total") + .unwrap(); + assert_eq!(observed.column_signs["value"], ValueSign::NonNegative); + assert_eq!( + observed.column_magnitudes["value"], + ObservedMagnitude { + min_abs: 2.0, + max_abs: 10.0, + } + ); +} + +/// A series of one sample has no gap and does not panic: the interval +/// comes from the other series. +#[test] +fn facts_single_sample_series() { + let samples = vec![ + sample("s", &[("id", "lonely")], 0, 1.0), + sample("s", &[("id", "p")], 0, 1.0), + sample("s", &[("id", "p")], 3_000, 1.0), + ]; + let workload = workload( + &[("max_over_time(s[10s])", per_value("absolute", 1.0, 0.01))], + 1_000, + &[], + ); + let observed = observe_source_statistics(&samples, &workload) + .remove("s") + .unwrap(); + assert_eq!(observed.sample_interval_ms, Some(3_000)); + let alone = vec![sample("t", &[], 0, 1.0)]; + let workload = workload_over("t"); + assert_eq!( + observe_source_statistics(&alone, &workload)["t"].sample_interval_ms, + None + ); +} + +fn workload_over(metric: &str) -> PlanningWorkload { + workload( + &[( + &format!("max_over_time({metric}[10s])"), + per_value("absolute", 1.0, 0.01), + )], + 1_000, + &[], + ) +} + +/// The JSONL schema: one `{labels, ts_ms, value}` object per line. +// provisional answer 18 +#[test] +fn facts_reads_jsonl_samples() { + let text = "{\"labels\": {\"__name__\": \"m\", \"job\": \"a\"}, \"ts_ms\": 1000, \"value\": 2.0}\n\ + {\"labels\": {\"__name__\": \"m\", \"job\": \"b\"}, \"ts_ms\": 2000, \"value\": -0.5}\n"; + let samples = read_samples_jsonl(text).unwrap(); + assert_eq!( + samples, + vec![ + sample("m", &[("job", "a")], 1_000, 2.0), + sample("m", &[("job", "b")], 2_000, -0.5), + ] + ); + let error = read_samples_jsonl("{\"labels\": {}, \"ts_ms\": 1}\n").unwrap_err(); + assert_eq!(error.line, 1); +} + +/// Generated data round-trips through facts: 50 series over `host` (100 +/// values) × `endpoint` (8 values) every 10 s give series count 50, +/// interval 10,000, at most 50 hosts and 8 endpoints. +#[test] +fn facts_datagen_round_trip() { + let description = TimeSeriesDescription { + labels: vec![ + SeriesLabelSpec { + name: "host".into(), + cardinality: 100, + skewness: 0.0, + }, + SeriesLabelSpec { + name: "endpoint".into(), + cardinality: 8, + skewness: 1.1, + }, + ], + series_num: 50, + start_ms: 0, + sample_interval_ms: 10_000, + duration_ms: 600_000, + value_distribution: DataDistribution::Uniform(UniformParameter { + lower_bound: 0.0, + upper_bound: 1.0, + seed: 0, + }), + counter: false, + seed: 42, + }; + let samples = samples_of_generated("g", &description.generate().unwrap()).unwrap(); + let observed = observe_source_statistics(&samples, &workload_over("g")) + .remove("g") + .unwrap(); + assert_eq!(observed.series_count, 50); + assert_eq!(observed.sample_interval_ms, Some(10_000)); + assert!(observed.distinct_values_per_label["host"] <= 50); + assert!(observed.distinct_values_per_label["endpoint"] <= 8); +} diff --git a/crates/types/src/test_support/mod.rs b/crates/types/src/test_support/mod.rs index c6343f5d7..695f07b76 100644 --- a/crates/types/src/test_support/mod.rs +++ b/crates/types/src/test_support/mod.rs @@ -9,3 +9,6 @@ pub mod forwarders; pub mod memo_builders; /// Operator expressions the memo builders and tests share. pub mod memo_fixtures; +/// Every plan over Stage 2's physical candidates, for the exhaustive +/// oracle. +pub mod physical_plan_enumeration; diff --git a/crates/types/src/test_support/physical_plan_enumeration.rs b/crates/types/src/test_support/physical_plan_enumeration.rs new file mode 100644 index 000000000..3bafa6b0e --- /dev/null +++ b/crates/types/src/test_support/physical_plan_enumeration.rs @@ -0,0 +1,169 @@ +//! Every plan over Stage 2's physical candidates +//! ([`PhysicalASAPMemoCandidates`]), by plain enumeration, for an +//! exhaustive oracle that checks plan selection against every +//! alternative. It is written from Stage 2's output alone and shares no +//! code with Stage 3. +//! +//! A plan picks one physical candidate for each memo group it reaches, +//! starting at the query roots' groups, and one admissible edge into each +//! input of each picked candidate, whose producer is the candidate picked +//! for the producer's group. Example: `sum_over_time(m[5m])` with an +//! exact query-time candidate and an ingestion-time chain candidate in the +//! root group, each reading the scan's single candidate, has two plans. +//! +//! One candidate per group: a plan that reads one group through two of +//! its candidates (say exact for one query and a sketch for another) is +//! not enumerated. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; + +/// One plan: the candidate picked per memo group, and the edge feeding +/// each input of each picked candidate. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct PhysicalPlanChoice { + pub candidate_of_group: BTreeMap, + /// Keyed by (consumer, input index of the consumer). + pub edge_into_input: BTreeMap<(PhysicalCandidateId, usize), PhysicalEdgeId>, +} + +impl PhysicalPlanChoice { + /// The picked candidates. + pub fn candidates(&self) -> BTreeSet { + self.candidate_of_group.values().copied().collect() + } + + /// The picked edges. + pub fn edges(&self) -> BTreeSet { + self.edge_into_input.values().copied().collect() + } +} + +/// The enumeration stopped: more than `limit` plans. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TooManyPlans { + pub limit: usize, +} + +/// What is left to pick in a partial plan. +#[derive(Debug, Clone, Copy)] +enum Pick { + /// A candidate for this group (a query root's). + Group(MemoGroupId), + /// An edge into input `.1` of candidate `.0`. + Input(PhysicalCandidateId, usize), +} + +/// Every plan over `physical`, at most `limit` of them. +pub fn every_physical_plan( + physical: &PhysicalASAPMemoCandidates, + limit: usize, +) -> Result, TooManyPlans> { + let mut root_groups = Vec::new(); + for query in physical.logical_candidates.memo.queries() { + for &group in query.root.operator_refs() { + if !root_groups.contains(&group) { + root_groups.push(group); + } + } + } + let mut candidates_in_group: BTreeMap> = BTreeMap::new(); + for (index, candidate) in physical.physical_candidates.iter().enumerate() { + candidates_in_group + .entry(candidate.logical_candidate_id.group) + .or_default() + .push(PhysicalCandidateId(index)); + } + let mut plans = Vec::new(); + let pending: Vec = root_groups.into_iter().rev().map(Pick::Group).collect(); + extend( + physical, + &candidates_in_group, + &mut PhysicalPlanChoice::default(), + pending, + &mut plans, + limit, + )?; + Ok(plans) +} + +/// The group of physical candidate `candidate`. +fn group_of(physical: &PhysicalASAPMemoCandidates, candidate: PhysicalCandidateId) -> MemoGroupId { + physical.physical_candidates[candidate.0] + .logical_candidate_id + .group +} + +/// `pending` with a pick for every input of `candidate` on top. +fn with_inputs_of( + physical: &PhysicalASAPMemoCandidates, + candidate: PhysicalCandidateId, + pending: &[Pick], +) -> Vec { + let mut next = pending.to_vec(); + next.extend( + (0..physical.input_count(candidate)) + .rev() + .map(|input| Pick::Input(candidate, input)), + ); + next +} + +/// Complete `choice` by every way of making the `pending` picks (the +/// last one first), adding each complete plan to `plans`. +fn extend( + physical: &PhysicalASAPMemoCandidates, + candidates_in_group: &BTreeMap>, + choice: &mut PhysicalPlanChoice, + mut pending: Vec, + plans: &mut Vec, + limit: usize, +) -> Result<(), TooManyPlans> { + let Some(pick) = pending.pop() else { + if plans.len() == limit { + return Err(TooManyPlans { limit }); + } + plans.push(choice.clone()); + return Ok(()); + }; + match pick { + Pick::Group(group) => { + if choice.candidate_of_group.contains_key(&group) { + return extend(physical, candidates_in_group, choice, pending, plans, limit); + } + for &candidate in candidates_in_group.get(&group).into_iter().flatten() { + choice.candidate_of_group.insert(group, candidate); + let next = with_inputs_of(physical, candidate, &pending); + extend(physical, candidates_in_group, choice, next, plans, limit)?; + choice.candidate_of_group.remove(&group); + } + Ok(()) + } + Pick::Input(consumer, input) => { + for &edge_id in physical.admissible_edges_into(consumer, input) { + let producer = physical.edge(edge_id).producer_candidate_id; + let group = group_of(physical, producer); + let newly_picked = match choice.candidate_of_group.get(&group) { + Some(&picked) if picked != producer => continue, + Some(_) => false, + None => true, + }; + choice.edge_into_input.insert((consumer, input), edge_id); + let next = if newly_picked { + choice.candidate_of_group.insert(group, producer); + with_inputs_of(physical, producer, &pending) + } else { + pending.clone() + }; + let result = extend(physical, candidates_in_group, choice, next, plans, limit); + if newly_picked { + choice.candidate_of_group.remove(&group); + } + choice.edge_into_input.remove(&(consumer, input)); + result?; + } + Ok(()) + } + } +} From 3b31953005410a1dc3bec1a1f7df1a02a3ae01f7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:02:16 +0000 Subject: [PATCH 2/2] test(devtools): asap_run machine key = architecture + CPU model + logical cores + memory type Memory size and hostname are not part of the key. Memory type comes from EDAC, else unknown-memory, overridable by --memory-type / ASAP_MEMORY_TYPE. Tests and API skeleton only. --- .../src/measurement/calibration_tables.rs | 4 +- crates/devtools/src/measurement/machine.rs | 55 +++++++-- .../tests/measurement/machine_and_tables.rs | 115 ++++++++++++++---- 3 files changed, 136 insertions(+), 38 deletions(-) diff --git a/crates/devtools/src/measurement/calibration_tables.rs b/crates/devtools/src/measurement/calibration_tables.rs index 07cdcc9c1..c61423bb4 100644 --- a/crates/devtools/src/measurement/calibration_tables.rs +++ b/crates/devtools/src/measurement/calibration_tables.rs @@ -5,8 +5,8 @@ //! Layout, under `ASAP_CALIBRATION_DIR` or `/calibration`: //! `/{machine.json, costs.json, accuracy.json, //! calibrate-run.json}`. Example: with `ASAP_CALIBRATION_DIR=/data/cal` on -//! machine `xeon-gold-6142-56c-251g`, the cost table is -//! `/data/cal/xeon-gold-6142-56c-251g/costs.json`, unless `--costs ` +//! machine `x86_64-intel-xeon-gold-6142-56c-registered-ddr4`, the cost table is +//! `/data/cal/x86_64-intel-xeon-gold-6142-56c-registered-ddr4/costs.json`, unless `--costs ` //! names another. use std::path::{Path, PathBuf}; diff --git a/crates/devtools/src/measurement/machine.rs b/crates/devtools/src/measurement/machine.rs index 2bb4af966..b5bae874b 100644 --- a/crates/devtools/src/measurement/machine.rs +++ b/crates/devtools/src/measurement/machine.rs @@ -1,36 +1,73 @@ //! The machine a calibration is measured on, and the key its tables are -//! stored under: CPU model, logical cores and memory, never the hostname. +//! stored under: CPU architecture, CPU model, logical cores and memory +//! type. Never the hostname and never the memory size. //! -//! Example: an "Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz" with 56 logical -//! cores and 263,724,000 KiB of memory is `xeon-gold-6142-56c-251g`. +//! Example: an "Intel(R) Xeon(R) CPU E5-2683 v3 @ 2.00GHz" on x86_64 with 56 +//! logical cores and DDR4 registered DIMMs is +//! `x86_64-intel-xeon-e5-2683-v3-56c-registered-ddr4`. use serde::{Deserialize, Serialize}; +/// The memory type of a machine whose DIMM types cannot be read (no EDAC, +/// as in a VM) and that has no override. +pub const UNKNOWN_MEMORY_TYPE: &str = "unknown-memory"; + /// What `machine.json` records. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MachineDescription { + /// `std::env::consts::ARCH`, for example `x86_64`. + pub cpu_architecture: String, pub cpu_model: String, pub logical_cores: u32, - /// `MemTotal`, in KiB. + /// The DIMM type, for example `Registered-DDR4`; DIMM types that differ + /// are the sorted unique values joined by `-`; `UNKNOWN_MEMORY_TYPE` + /// when unreadable and not overridden. + pub memory_type: String, + /// `MemTotal`, in KiB. Recorded for information; not part of the key. pub memory_kib: u64, pub os: String, } impl MachineDescription { - /// The machine `/proc/cpuinfo` and `/proc/meminfo` describe. - pub fn from_proc(cpuinfo: &str, meminfo: &str, os: &str) -> Result { - let _ = (cpuinfo, meminfo, os); + /// The machine `/proc/cpuinfo`, `/proc/meminfo` and the EDAC DIMM types + /// (`/sys/devices/system/edac/mc/mc*/dimm*/dimm_mem_type`, one string per + /// DIMM, empty without EDAC) describe. `memory_type_override` + /// (`--memory-type` / `ASAP_MEMORY_TYPE`) replaces the EDAC types. + pub fn from_proc( + cpu_architecture: &str, + cpuinfo: &str, + meminfo: &str, + dimm_memory_types: &[&str], + memory_type_override: Option<&str>, + os: &str, + ) -> Result { + let _ = ( + cpu_architecture, + cpuinfo, + meminfo, + dimm_memory_types, + memory_type_override, + os, + ); todo!("asap_run: implement after test approval") } /// This machine. - pub fn detect() -> Result { + pub fn detect(memory_type_override: Option<&str>) -> Result { + let _ = memory_type_override; todo!("asap_run: implement after test approval") } - /// The directory name its tables are stored under: `[a-z0-9-]+`. + /// The directory name its tables are stored under: `[a-z0-9-]+`, from + /// architecture, model, cores and memory type. pub fn machine_key(&self) -> String { todo!("asap_run: implement after test approval") } + + /// A warning when the memory type is `UNKNOWN_MEMORY_TYPE`, naming it + /// and the `--memory-type` / `ASAP_MEMORY_TYPE` override. + pub fn warning(&self) -> Option { + todo!("asap_run: implement after test approval") + } } diff --git a/crates/devtools/tests/measurement/machine_and_tables.rs b/crates/devtools/tests/measurement/machine_and_tables.rs index a44cf0608..7b753e6dc 100644 --- a/crates/devtools/tests/measurement/machine_and_tables.rs +++ b/crates/devtools/tests/measurement/machine_and_tables.rs @@ -14,8 +14,7 @@ use asap_planner::DeploymentCapabilities; use super::common::{exact, facts, plan, workload, TempDir}; -const XEON_CPUINFO: &str = "processor\t: 0\nvendor_id\t: GenuineIntel\n\ -model name\t: Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz\n\n"; +const E5_MODEL: &str = "Intel(R) Xeon(R) CPU E5-2683 v3 @ 2.00GHz"; /// `/proc/cpuinfo` of `cores` logical cores of `model`. fn cpuinfo(model: &str, cores: u32) -> String { @@ -28,40 +27,54 @@ fn meminfo(kib: u64) -> String { format!("MemTotal: {kib} kB\nMemFree: 1000 kB\n") } -fn xeon(memory_kib: u64) -> MachineDescription { +/// 56 logical cores of the E5-2683 v3 on x86_64 with `memory_kib` of memory, +/// DIMM types `dimm_types` (EDAC) and an optional override. +fn e5( + memory_kib: u64, + dimm_types: &[&str], + memory_type_override: Option<&str>, +) -> MachineDescription { MachineDescription::from_proc( - &cpuinfo("Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz", 56), + "x86_64", + &cpuinfo(E5_MODEL, 56), &meminfo(memory_kib), + dimm_types, + memory_type_override, "Linux 5.15", ) .unwrap() } -/// 56 logical cores of a Xeon Gold 6142 with 263,724,000 KiB: -/// 263,724,000 / 2^20 = 251.5 GiB, floored to 251. -// provisional answer 16 +const E5_MEMORY_KIB: u64 = 263_724_000; +const E5_KEY: &str = "x86_64-intel-xeon-e5-2683-v3-56c-registered-ddr4"; + +/// Architecture, model, cores and memory type give the slug: `(R)`, `CPU` +/// and `@ 2.00GHz` are dropped from the model, the rest is lowercased and +/// dash-joined. +// decided 2026-11-16: key = architecture + model + logical cores + memory type #[test] fn machine_key_from_cpu_cores_memory() { - assert!(XEON_CPUINFO.contains("6142")); - let machine = xeon(263_724_000); + let machine = e5(E5_MEMORY_KIB, &["Registered-DDR4", "Registered-DDR4"], None); + assert_eq!(machine.cpu_architecture, "x86_64"); assert_eq!(machine.logical_cores, 56); - assert_eq!(machine.memory_kib, 263_724_000); - assert_eq!(machine.machine_key(), "xeon-gold-6142-56c-251g"); + assert_eq!(machine.memory_type, "Registered-DDR4"); + assert_eq!(machine.memory_kib, E5_MEMORY_KIB); + assert_eq!(machine.machine_key(), E5_KEY); } -/// The key is a function of CPU model, cores and memory only: this -/// machine's detected key contains no token of its hostname that the CPU -/// model does not also contain, and the description has no hostname to -/// change. +/// The key is a function of architecture, CPU model, cores and memory type +/// only: this machine's detected key contains no token of its hostname that +/// the CPU model does not also contain, and the description has no hostname +/// to change. #[test] fn machine_key_ignores_hostname() { - let fixture = xeon(263_724_000).machine_key(); - for hostname in ["a", "xeon-gold-6142", "node-17"] { + let fixture = e5(E5_MEMORY_KIB, &["Registered-DDR4"], None).machine_key(); + for hostname in ["a", "xeon-e5-2683", "node-17"] { // The fixture's inputs do not include a hostname, so every // hostname gives the same key, and it is never the hostname. assert_ne!(fixture, hostname); } - let detected = MachineDescription::detect().unwrap(); + let detected = MachineDescription::detect(None).unwrap(); let key = detected.machine_key(); let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default(); let model = detected.cpu_model.to_lowercase(); @@ -79,23 +92,71 @@ fn machine_key_ignores_hostname() { } } -/// `MemTotal` jitters by about ±0.5% across boots: the key does not change. -// provisional answer 16: GiB rounding cannot hold this for the 251.5 GiB -// fixture (+0.5% is 252.8 GiB, −0.5% is 250.2 GiB); the test is written -// as the plan states it so the conflict is visible. +/// Total memory is not part of the key: 64 GiB and 256 GiB of the same +/// DIMM type give the same key, and `MemTotal` jitter across boots too. +#[test] +fn machine_key_ignores_memory_size() { + let dimms = ["Registered-DDR4"]; + let key = e5(E5_MEMORY_KIB, &dimms, None).machine_key(); + assert_eq!(key, E5_KEY); + assert_eq!(e5(64 * 1024 * 1024, &dimms, None).machine_key(), key); + assert_eq!(e5(256 * 1024 * 1024, &dimms, None).machine_key(), key); + assert_eq!( + e5(E5_MEMORY_KIB * 1005 / 1000, &dimms, None).machine_key(), + key + ); +} + +/// A VM without EDAC and without an override: `unknown-memory` ends the +/// key, and the warning names it and the override. +#[test] +fn machine_key_without_edac_is_unknown_memory() { + let machine = e5(E5_MEMORY_KIB, &[], None); + assert_eq!( + machine.machine_key(), + "x86_64-intel-xeon-e5-2683-v3-56c-unknown-memory" + ); + let warning = machine.warning().expect("a warning for unknown memory"); + assert!(warning.contains("unknown-memory"), "{warning}"); + assert!(warning.contains("ASAP_MEMORY_TYPE"), "{warning}"); + assert!(e5(E5_MEMORY_KIB, &["Registered-DDR4"], None) + .warning() + .is_none()); +} + +/// `--memory-type` / `ASAP_MEMORY_TYPE` supplies the type when EDAC has +/// none, and no warning is given. +#[test] +fn machine_key_memory_type_override() { + let machine = e5(E5_MEMORY_KIB, &[], Some("Registered-DDR4")); + assert_eq!(machine.machine_key(), E5_KEY); + assert!(machine.warning().is_none()); +} + +/// DIMMs of different types: the sorted unique values joined, whatever the +/// DIMM order. #[test] -fn machine_key_stable_under_memtotal_jitter() { - let key = xeon(263_724_000).machine_key(); - assert_eq!(xeon(263_724_000 * 1005 / 1000).machine_key(), key); - assert_eq!(xeon(263_724_000 * 995 / 1000).machine_key(), key); +fn machine_key_mixed_dimm_types() { + for dimms in [ + ["Registered-DDR4", "LRDIMM-DDR4", "Registered-DDR4"], + ["LRDIMM-DDR4", "Registered-DDR4", "LRDIMM-DDR4"], + ] { + assert_eq!( + e5(E5_MEMORY_KIB, &dimms, None).machine_key(), + "x86_64-intel-xeon-e5-2683-v3-56c-lrdimm-ddr4-registered-ddr4" + ); + } } /// Only `[a-z0-9-]`, no leading, trailing or double dashes. #[test] fn machine_key_slug_charset() { let machine = MachineDescription::from_proc( + "aarch64", &cpuinfo("AMD EPYC(TM) 7452 32-Core Processor @ 2.35GHz (R)", 8), &meminfo(16_000_000), + &["Registered-DDR4"], + None, "Linux", ) .unwrap(); @@ -111,7 +172,7 @@ fn machine_key_slug_charset() { ); } -const KEY: &str = "xeon-gold-6142-56c-251g"; +const KEY: &str = E5_KEY; /// `--costs` first, then `ASAP_CALIBRATION_DIR`, then `/calibration`, /// then nothing (the formulas).