diff --git a/crates/integration-tests/tests/cost_oracle.rs b/crates/integration-tests/tests/cost_oracle.rs new file mode 100644 index 000000000..44bf0f1a2 --- /dev/null +++ b/crates/integration-tests/tests/cost_oracle.rs @@ -0,0 +1,356 @@ +//! Hand-computed costs for the canonical plan shapes +//! (`docs/develop_docs/testing.md`, "Oracles"): each expected cost is +//! computed here from the concepts doc's cost terms (section 5, "Cost and +//! retention": executions per second ρ_κ times one execution's work, and +//! a materialized node's retained state), not by `asap_cost`'s node +//! costing, and compared with `asap_cost::plan_cost` of the plan Stage 3 +//! selects. +//! +//! The plans are costed under cost inputs that charge one resource +//! ([`charging_only`]): [`CPU_OPERATION`] charges 1 per CPU operation, so a +//! node's cost is its CPU operations per second; [`RETAINED_BYTE_SECOND`] +//! charges 1 per byte held for a second. Two terms are the cost model's +//! (`asap-cost`, module `analytical`), stated here and not checked: +//! +//! - the per-operation work: a scan, a time range and a merge do one +//! operation per row or state they produce or read, a finalize one per +//! accumulator, a hash aggregate one per input row and per grouping key +//! and accumulator, an exact accumulator build one per row and per update +//! (depth 1), a time shift none; +//! - a chain retains, beside the windows its consumers read (offset + range +//! + span, h_v), the one window being built. +//! +//! The counts are what this oracle checks: rows per execution, windows a +//! range selects, states a merge reads, executions per second and retained +//! windows. Each expected value is a literal or arithmetic over the named +//! constants; nothing is read from the planner but the plan's costs. +//! +//! Each test checks that Stage 3 selects the canonical shape before costing +//! it: the oracle's counts are for that shape only, so a change of the +//! selected plan fails with "not the canonical shape", not a wrong count. + +use asap_cost::{plan_cost, CostInputs, CostModel, ResourceUnitCost}; +use asap_devtools::corpus_registry::{plan_workload, repeating_promql_workload, PlannedWorkload}; +use asap_devtools::planning_stages::RunUntil; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::physical_export::PhysicalASAPDAG; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::workload::{DurationMs, PrometheusMetricType, RangeEvaluation, SourceFacts}; + +/// Series of the one metric `m`, each sampled every second: 1,000 rows +/// arrive per second. +const SERIES: f64 = 1_000.0; +const ROWS_PER_SECOND: f64 = 1_000.0; +/// Every query repeats every 60 s. +const EVALUATIONS_PER_SECOND: f64 = 1.0 / 60.0; +/// The bytes of one exact `sum` or `rate` accumulator state (a sum and a +/// count with their group key), as `asap_cost::state_bytes` sizes it. +const EXACT_ACCUMULATOR_STATE_BYTES: f64 = 56.0; +/// `sum(..)`'s grouping keys and accumulators: a hash aggregate does one +/// operation per input row per (key + accumulator). +const GROUPING_KEYS: f64 = 0.0; +const ACCUMULATORS: f64 = 1.0; + +/// No resource charged. +const NOTHING: ResourceUnitCost = ResourceUnitCost { + cost_per_cpu_op: 0.0, + cost_per_scanned_byte: 0.0, + cost_per_memory_byte_second: 0.0, + cost_per_provisioned_memory_byte_second: 0.0, + cost_per_provisioned_cpu_core_second: 0.0, + cost_per_disk_byte_second: 0.0, +}; +/// 1 per CPU operation and nothing else. +const CPU_OPERATION: ResourceUnitCost = ResourceUnitCost { + cost_per_cpu_op: 1.0, + ..NOTHING +}; +/// 1 per byte held in memory for a second and nothing else. +const RETAINED_BYTE_SECOND: ResourceUnitCost = ResourceUnitCost { + cost_per_memory_byte_second: 1.0, + ..NOTHING +}; + +/// Cost model 1 with `unit_costs` (one resource charged). +fn charging_only(unit_costs: ResourceUnitCost) -> CostInputs { + CostInputs { + cost_model: CostModel::UsageOverTime(unit_costs), + ..CostInputs::default() + } +} + +/// `query` over the one metric `m` with facts `source`, repeating every +/// 60 s at phase 0 at `accuracy` (a range query when `range_evaluation` is +/// set), planned to Stage 3 on `capabilities` with the default cost inputs. +/// The oracles' one planning helper. +pub(crate) fn plan_one_promql_query( + query: &str, + accuracy: AccuracyTarget, + source: SourceFacts, + range_evaluation: Option, + capabilities: &DeploymentCapabilities, +) -> PlannedWorkload { + let mut workload = repeating_promql_workload(&[query], 60_000, accuracy, &[("m", source)]); + workload.query_workload.repeating_queries.as_mut().unwrap()[0].range_evaluation = + range_evaluation; + plan_workload( + &workload, + &[], + capabilities, + &CostInputs::default(), + false, + RunUntil::Stage3, + ) + .unwrap() +} + +/// The operator of each of `plan`'s nodes, in node order. +pub(crate) fn operator_kinds(plan: &PhysicalASAPDAG) -> Vec<&'static str> { + plan.nodes + .iter() + .map(|node| node.payload.kind_name()) + .collect() +} + +/// `query` exactly, over `m`: 1,000 series of `metric_type` sampled every +/// second. +fn planned( + query: &str, + metric_type: PrometheusMetricType, + range_evaluation: Option, + capabilities: &DeploymentCapabilities, +) -> PlannedWorkload { + let source = SourceFacts { + series_count: Some(SERIES as u64), + sample_interval: Some(DurationMs(1_000)), + metric_type: Some(metric_type), + ..SourceFacts::default() + }; + plan_one_promql_query( + query, + AccuracyTarget::exact(), + source, + range_evaluation, + capabilities, + ) +} + +/// The selected plan, checked to have the operators `shape` (one per node, +/// in node order): a plan of another shape is not what the oracle computes. +fn plan_of_shape<'a>(planned: &'a PlannedWorkload, shape: &[&str]) -> &'a PhysicalASAPDAG { + let (plan, _) = planned + .stages + .run + .selection + .as_ref() + .expect("a plan is selected"); + assert_eq!( + operator_kinds(plan), + shape, + "the selected plan is not the canonical shape" + ); + plan +} + +/// Each node's cost per second, in node order, and the plan's, of `plan` +/// under `inputs`. +fn costs( + plan: &PhysicalASAPDAG, + planned: &PlannedWorkload, + inputs: &CostInputs, +) -> (Vec, f64) { + let (cost, _) = plan_cost(plan, &planned.entries, &planned.data, inputs).unwrap(); + let per_node = plan + .nodes + .iter() + .map(|node| cost.per_node[&node.id].cost_per_second.unwrap()) + .collect(); + (per_node, cost.cost_per_second.unwrap()) +} + +/// `actual` equals `expected` to a relative 1e-9. +fn assert_close(what: &str, actual: f64, expected: f64) { + assert!( + (actual - expected).abs() <= 1e-9 * expected.abs().max(1e-12), + "{what}: plan_cost says {actual}, the oracle {expected}" + ); +} + +/// `sum(m)` every 60 s on the default deployment: a query-time `Scan` +/// reads one second of rows (the instant selector's lookback, the sample +/// interval), 1,000 rows; the instant `TimeRange` keeps the latest row of +/// each of the 1,000 series; the aggregate reads 1,000 rows into one +/// group. Each runs 1/60 times per second: 50 operations per second. +#[test] +fn a_query_time_scan_costs_the_rows_its_lookback_reads() { + let planned = planned( + "sum(m)", + PrometheusMetricType::Gauge, + None, + &DeploymentCapabilities::default(), + ); + let plan = plan_of_shape(&planned, &["Scan", "TimeRange", "Aggregate"]); + let rows_read = ROWS_PER_SECOND * 1.0; + let expected = [ + rows_read * EVALUATIONS_PER_SECOND, + SERIES * EVALUATIONS_PER_SECOND, + SERIES * (GROUPING_KEYS + ACCUMULATORS) * EVALUATIONS_PER_SECOND, + ]; + let (per_node, total) = costs(plan, &planned, &charging_only(CPU_OPERATION)); + for (node, (actual, expected)) in per_node.iter().zip(expected).enumerate() { + assert_close(&format!("node {node}"), *actual, expected); + } + assert_close("plan", total, expected.iter().sum()); +} + +/// The plan of `sum(sum_over_time(m[5m]))` every 60 s on a deployment that +/// keeps no raw data: an ingestion-time `Scan`, a chain of 60 s windows +/// of exact sums materialized across evaluations, then at query time a 5 +/// min `TimeRange` over the chain, a `SummaryMerge`, the finalize and the +/// sum. +fn chain_plan() -> PlannedWorkload { + let no_raw_data = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + planned( + "sum(sum_over_time(m[5m]))", + PrometheusMetricType::Gauge, + None, + &no_raw_data, + ) +} + +const CHAIN_SHAPE: [&str; 6] = [ + "Scan", + "SummaryTumblingWindowChain", + "TimeRange", + "SummaryMerge", + "FinalizeExactAccumulator", + "Aggregate", +]; + +/// At ingestion time each second's 1,000 arriving rows are scanned once +/// (1,000 operations per second) and each updates one window's exact sum +/// state (depth 1, plus the row: 2,000 per second). The chain retains the +/// 5 windows its 5 min consumer reads (h_v = 5 min of 60 s windows) plus +/// the window being built: 6 × 1,000 states of 56 bytes, 336,000 bytes +/// held. +#[test] +fn an_ingestion_time_chain_costs_its_arriving_rows_and_retains_its_windows() { + let planned = chain_plan(); + let plan = plan_of_shape(&planned, &CHAIN_SHAPE); + let (cpu, _) = costs(plan, &planned, &charging_only(CPU_OPERATION)); + assert_close("ingestion-time Scan", cpu[0], ROWS_PER_SECOND); + assert_close("chain build", cpu[1], ROWS_PER_SECOND * (1.0 + 1.0)); + let (memory, _) = costs(plan, &planned, &charging_only(RETAINED_BYTE_SECOND)); + let retained_windows = 300.0 / 60.0; + let window_being_built = 1.0; + assert_close( + "chain retention", + memory[1], + (retained_windows + window_being_built) * SERIES * EXACT_ACCUMULATOR_STATE_BYTES, + ); +} + +/// The 5 min `TimeRange` over a chain of 60 s windows selects 5 windows of +/// 1,000 states, 5,000 states per evaluation, 1/60 evaluations per second. +#[test] +fn a_time_range_over_a_chain_selects_the_windows_its_range_covers() { + let planned = chain_plan(); + let plan = plan_of_shape(&planned, &CHAIN_SHAPE); + let (cpu, _) = costs(plan, &planned, &charging_only(CPU_OPERATION)); + let windows = 300.0 / 60.0; + assert_close( + "TimeRange over the chain", + cpu[2], + windows * SERIES * EVALUATIONS_PER_SECOND, + ); +} + +/// The `SummaryMerge` reads every selected window's states (5 × 1,000) +/// into one per series (1,000), which the finalize reads and the sum +/// aggregates; the plan's CPU is 1,000 + 2,000 per second at ingestion +/// plus (5,000 + 5,000 + 1,000 + 1,000) / 60 at query time. +#[test] +fn a_summary_merge_reads_every_selected_window_state() { + let planned = chain_plan(); + let plan = plan_of_shape(&planned, &CHAIN_SHAPE); + let (cpu, total) = costs(plan, &planned, &charging_only(CPU_OPERATION)); + let states = 5.0 * SERIES; + assert_close("SummaryMerge", cpu[3], states * EVALUATIONS_PER_SECOND); + assert_close("finalize", cpu[4], SERIES * EVALUATIONS_PER_SECOND); + assert_close( + "sum", + cpu[5], + SERIES * (GROUPING_KEYS + ACCUMULATORS) * EVALUATIONS_PER_SECOND, + ); + assert_close( + "plan", + total, + ROWS_PER_SECOND * 3.0 + (states + states + SERIES + SERIES) * EVALUATIONS_PER_SECOND, + ); +} + +/// #1253 Example 1, o11y q21's shape: `sum(rate(m[5m] offset 1h))` as a +/// 1 h range query (step 60 s) refreshed every 60 s over a chain of 60 s +/// windows of exact rate states. Each refresh evaluates 61 steps, so the +/// query-time nodes run 61/60 times per second. At each step the 5 min +/// `TimeRange` over the 1 h `TimeShift` selects 5 windows of 1,000 +/// states and the merge reads those 5,000 states into 1,000 (before the +/// fix, the shift hid the chain: one window, "merge 1000 states into +/// 1000"). The chain retains offset + range + span = 1 h + 5 min + 1 h = +/// 125 windows, plus the one being built. +#[test] +fn a_time_range_over_a_shifted_chain_merges_every_selected_window() { + let planned = planned( + "sum(rate(m[5m] offset 1h))", + PrometheusMetricType::Counter, + Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }), + &DeploymentCapabilities::default(), + ); + let plan = plan_of_shape( + &planned, + &[ + "Scan", + "SummaryTumblingWindowChain", + "TimeShift", + "TimeRange", + "SummaryMerge", + "FinalizeExactAccumulator", + "Aggregate", + ], + ); + let steps_per_refresh = 3_600.0 / 60.0 + 1.0; + let query_time_executions_per_second = steps_per_refresh * EVALUATIONS_PER_SECOND; + let states = 5.0 * SERIES; + let expected = [ + ROWS_PER_SECOND, + ROWS_PER_SECOND * (1.0 + 1.0), + 0.0, + states * query_time_executions_per_second, + states * query_time_executions_per_second, + SERIES * query_time_executions_per_second, + SERIES * (GROUPING_KEYS + ACCUMULATORS) * query_time_executions_per_second, + ]; + let (cpu, total) = costs(plan, &planned, &charging_only(CPU_OPERATION)); + for (node, (actual, expected)) in cpu.iter().zip(expected).enumerate() { + assert_close( + &format!("node {node} ({})", plan.nodes[node].payload.kind_name()), + *actual, + expected, + ); + } + assert_close("plan", total, expected.iter().sum()); + let (memory, _) = costs(plan, &planned, &charging_only(RETAINED_BYTE_SECOND)); + let retained_windows = (3_600.0 + 300.0 + 3_600.0) / 60.0; + let window_being_built = 1.0; + assert_close( + "chain retention", + memory[1], + (retained_windows + window_being_built) * SERIES * EXACT_ACCUMULATOR_STATE_BYTES, + ); +} diff --git a/crates/integration-tests/tests/execution_oracle.rs b/crates/integration-tests/tests/execution_oracle.rs new file mode 100644 index 000000000..08c0beca8 --- /dev/null +++ b/crates/integration-tests/tests/execution_oracle.rs @@ -0,0 +1,565 @@ +//! The execution oracle (`docs/develop_docs/testing.md`, "Oracles"): the +//! plan Stage 3 selects is run by the executor over synthetic data, as a +//! runtime runs it (every materialized view refreshed over its windows, +//! then the query run), and its answers are compared with an exact +//! evaluation of the query written here over the same samples. Exact +//! queries must give equal answers; an approximate query's answers may be +//! off by more than ε for at most a fraction δ of its values, checked over +//! every series' value with a fixed seed. +//! +//! Each test checks that the plan takes the path it means to execute (a +//! chain, a sketch): the answers of another plan would not test that path. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use asap_executor::physical_planner::compile; +use asap_executor::runtime::{Limits, Scope}; +use asap_executor::sources::InMemoryDataStore; +use asap_executor::values::{Batch, Value}; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::materialized_view::MaterializedViewRefresh; +use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; +use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; +use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; +use asap_types::ir::ASAPOp; +use asap_types::stages::ValueSign; +use asap_types::workload::{DurationMs, PrometheusMetricType, SourceFacts}; + +use crate::cost_oracle::{operator_kinds, plan_one_promql_query}; +use crate::physical_common::{execute, raw_data_store}; + +/// The evaluation time every query is answered at: 30 min of samples +/// before it (the longest window read is 15 min back), on a 60 s +/// boundary. +const EVALUATION_MS: i64 = 1_800_000; +/// Series of `m`, each labelled `instance="i"`. +const SERIES: usize = 200; +/// Every series has a sample every second from 0 up to [`EVALUATION_MS`]. +const SAMPLE_INTERVAL_MS: i64 = 1_000; +/// The fixed seed of the sample values. +const SEED: u64 = 1253; +/// Each sample's value is uniform in [0, [`NOISE_RANGE`]) plus +/// [`DRIFT_PER_MINUTE`] times the minute it falls in, so every 60 s window +/// holds values of its own range: an answer read from the wrong windows +/// moves by far more than any ε. +const NOISE_RANGE: f64 = 100.0; +const DRIFT_PER_MINUTE: f64 = 10.0; +/// The raw data is one 360,000-row batch, above the executor's default +/// 64 MiB per run. +const MAX_BYTES_PER_RUN: usize = 1 << 30; + +/// One series of `m`: its `instance` label, all its labels, and its +/// identity column's value (the labels as JSON), encoded once. +#[derive(Debug)] +struct Series { + instance: String, + labels: BTreeMap>, + identity: Arc, +} + +/// One raw data sample: its series, time and value. +#[derive(Debug, Clone)] +struct Sample { + series: Arc, + timestamp_ms: i64, + value: f64, +} + +/// A xorshift64* generator: the samples depend only on [`SEED`]. +struct Rng(u64); + +impl Rng { + /// A value uniform in [0, 1). + fn next_unit(&mut self) -> f64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + let bits = self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 11; + bits as f64 / (1u64 << 53) as f64 + } +} + +/// [`SERIES`] series of `m`, a sample every second in [0, EVALUATION_MS] +/// (a non-negative gauge whose values drift up each minute, see +/// [`DRIFT_PER_MINUTE`]), sorted by time. +fn samples() -> Vec { + let mut rng = Rng(SEED); + let mut samples = Vec::new(); + for series in 0..SERIES { + let labels = BTreeMap::from([ + ("__name__".to_string(), "m".to_string()), + ("instance".to_string(), format!("i{series}")), + ]); + let series = Arc::new(Series { + instance: labels["instance"].clone(), + identity: serde_json::to_string(&labels).unwrap().into(), + labels: labels + .into_iter() + .map(|(name, value)| (name, value.into())) + .collect(), + }); + for step in 0..=(EVALUATION_MS / SAMPLE_INTERVAL_MS) { + let timestamp_ms = step * SAMPLE_INTERVAL_MS; + let minute = (timestamp_ms / 60_000) as f64; + samples.push(Sample { + series: series.clone(), + timestamp_ms, + value: rng.next_unit() * NOISE_RANGE + DRIFT_PER_MINUTE * minute, + }); + } + } + samples.sort_by_key(|sample| sample.timestamp_ms); + samples +} + +/// The samples of `samples` (sorted by time) with times in `(start, end]`. +fn between(samples: &[Sample], start_ms: i64, end_ms: i64) -> &[Sample] { + let from = samples.partition_point(|sample| sample.timestamp_ms <= start_ms); + let to = samples.partition_point(|sample| sample.timestamp_ms <= end_ms); + &samples[from..to] +} + +/// Each series' sample values with times in `(EVALUATION_MS - offset - +/// range, EVALUATION_MS - offset]`, by its `instance`: PromQL's range +/// selector `m[range] offset offset`. +fn window(samples: &[Sample], range_ms: i64, offset_ms: i64) -> BTreeMap> { + let end = EVALUATION_MS - offset_ms; + let mut windows: BTreeMap> = BTreeMap::new(); + for sample in between(samples, end - range_ms, end) { + windows + .entry(sample.series.instance.clone()) + .or_default() + .push(sample.value); + } + windows +} + +/// `m`'s facts: `series_count` series (declared; the data has [`SERIES`]) +/// of a non-negative gauge sampled every second. +fn source(series_count: u64) -> SourceFacts { + SourceFacts { + series_count: Some(series_count), + sample_interval: Some(DurationMs(SAMPLE_INTERVAL_MS as u64)), + metric_type: Some(PrometheusMetricType::Gauge), + column_signs: BTreeMap::from([("value".to_string(), ValueSign::NonNegative)]), + ..SourceFacts::default() + } +} + +/// The plan Stage 3 selects for `query` every 60 s at `accuracy` over `m` +/// declared with `series_count` series, on `capabilities`. +fn plan( + query: &str, + accuracy: AccuracyTarget, + series_count: u64, + capabilities: &DeploymentCapabilities, +) -> PhysicalASAPDAG { + let planned = plan_one_promql_query(query, accuracy, source(series_count), None, capabilities); + let (plan, _) = planned.stages.run.selection.expect("a plan is selected"); + plan +} + +/// A deployment that keeps no raw data: every window is read from state +/// built at ingestion time. +fn no_raw_data() -> DeploymentCapabilities { + DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + } +} + +/// `sample` as a row of `schema`: its identity, time, value and labels. +fn row(schema: &Schema, sample: &Sample) -> Vec { + schema + .fields + .iter() + .enumerate() + .map(|(index, field)| match field.name.as_str() { + PROMQL_SERIES_IDENTITY => Value::Utf8(sample.series.identity.clone()), + _ if Some(index) == schema.time_index => Value::Timestamp(sample.timestamp_ms), + "value" if field.dtype == FieldDataType::Plain(DataType::Float64) => { + Value::Float64(sample.value) + } + label => Value::Utf8( + sample + .series + .labels + .get(label) + .cloned() + .unwrap_or_else(|| "".into()), + ), + }) + .collect() +} + +/// One stored state of a view and the `(start, end]` of the data it holds. +struct ViewState { + batch: Batch, + start_ms: i64, + end_ms: i64, +} + +/// Runs node `id` of `dag` in `scope`, `raw` bound at its `Scan`s of `m` +/// (none bound when `raw` is `None`: a run that reads raw data then fails) +/// and each view of `views` bound to its stored states: the node's output +/// rows. +fn run( + dag: &PhysicalASAPDAG, + raw: Option<&[Sample]>, + views: &BTreeMap>, + id: u64, + scope: Scope, +) -> (Arc, Vec>) { + let bound: Vec<_> = dag + .materialized_views() + .unwrap() + .into_iter() + .filter(|view| views.contains_key(&(view.node_id as u64))) + .collect(); + let compiled = compile(dag, &bound, &[id]).expect("the executor compiles the plan"); + let mut store = match raw { + Some(raw) => raw_data_store(dag, |_, schema| { + raw.iter().map(|sample| row(schema, sample)).collect() + }), + None => InMemoryDataStore::default(), + }; + for view in &bound { + for state in &views[&(view.node_id as u64)] { + store + .append_materialized_view( + view, + (state.start_ms, state.end_ms), + vec![state.batch.clone()], + ) + .unwrap(); + } + } + let limits = Limits { + max_bytes: MAX_BYTES_PER_RUN, + ..Limits::default() + }; + let batches = execute(&compiled, store, scope, limits).remove(0); + let schema = Arc::new(dag.nodes[id as usize].output_schema.clone()); + (schema, batches.iter().flat_map(|b| b.to_rows()).collect()) +} + +/// The plan's answer at [`EVALUATION_MS`] as a runtime computes it on +/// `deployment`: every materialized view refreshed in plan order (a chain +/// once per window its retention keeps, newest ending at the evaluation +/// time, in an ingestion run over the samples of that window and the +/// windows beside it), then the query run reading them, bound the raw data +/// only when the deployment keeps it. Each answer row's value (its +/// `value_column`: `value` for a per-series function, the aggregate's name +/// for an aggregate) by its series' `instance` (`""` for an aggregate over +/// every series); two rows of one series fail. +fn answers( + dag: &PhysicalASAPDAG, + samples: &[Sample], + deployment: &DeploymentCapabilities, + value_column: &str, +) -> BTreeMap { + let mut built: BTreeMap> = BTreeMap::new(); + for view in dag.materialized_views().unwrap() { + let node = &dag.nodes[view.node_id]; + let width_ms = match &node.payload { + PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => *tumbling_window_width_ms as i64, + other => panic!( + "a materialized view other than a chain: {}", + other.kind_name() + ), + }; + let windows = (view.retention_ms() as i64 / width_ms).max(1); + let mut states = Vec::new(); + for k in (0..windows).rev() { + let end_ms = EVALUATION_MS - k * width_ms; + let start_ms = end_ms - width_ms; + let scope = match view.refresh() { + MaterializedViewRefresh::AtIngestion => Scope::Ingestion { + window_start_ms: start_ms, + window_end_ms: end_ms, + }, + MaterializedViewRefresh::OnFirstRead => Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }, + }; + // An ingestion run is bound the samples of its window and of the + // windows beside it (binding all of them only slows the run). + let near = between(samples, start_ms - width_ms, end_ms + width_ms); + let (schema, rows) = run(dag, Some(near), &built, view.node_id as u64, scope); + states.push(ViewState { + batch: Batch::try_new(schema, rows).unwrap(), + start_ms, + end_ms, + }); + } + built.insert(view.node_id as u64, states); + } + assert_eq!(dag.roots.len(), 1, "one query"); + let query = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + let raw = deployment.raw_data_retained.then_some(samples); + let (schema, rows) = run(dag, raw, &built, dag.roots[0] as u64, query); + let value_column = schema + .fields + .iter() + .position(|field| field.name == value_column) + .unwrap_or_else(|| panic!("no column {value_column} in {:?}", schema.fields)); + let mut answers = BTreeMap::new(); + for row in &rows { + let mut instance = String::new(); + for (field, cell) in schema.fields.iter().zip(row) { + match cell { + Value::Utf8(text) if field.name == PROMQL_SERIES_IDENTITY => { + let labels: BTreeMap = serde_json::from_str(text).unwrap(); + instance = labels.get("instance").cloned().unwrap_or_default(); + } + Value::Utf8(text) if field.name == "instance" => instance = text.to_string(), + _ => {} + } + } + let value = match &row[value_column] { + Value::Float64(value) => *value, + Value::Int64(value) => *value as f64, + other => panic!("a non-numeric answer {other:?}"), + }; + if let Some(earlier) = answers.insert(instance.clone(), value) { + panic!("two answers for series {instance:?}: {earlier} and {value}"); + } + } + answers +} + +/// `actual` equals `expected` per series, to a relative 1e-9. +fn assert_answers_equal(actual: &BTreeMap, expected: &BTreeMap) { + assert_eq!( + actual.keys().collect::>(), + expected.keys().collect::>(), + "the plan answers other series than the exact evaluation" + ); + for (series, value) in expected { + let got = actual[series]; + assert!( + (got - value).abs() <= 1e-9 * value.abs().max(1.0), + "series {series:?}: the plan answers {got}, the exact evaluation {value}" + ); + } +} + +/// `sum(m)` (exact, query-time scan of the latest sample of each series): +/// the sum of the 200 series' samples at the evaluation time. +#[test] +fn an_exact_instant_sum_equals_the_sum_of_the_latest_samples() { + let deployment = DeploymentCapabilities::default(); + let plan = plan( + "sum(m)", + AccuracyTarget::exact(), + SERIES as u64, + &deployment, + ); + let samples = samples(); + let expected: f64 = window(&samples, SAMPLE_INTERVAL_MS, 0) + .values() + .map(|v| v[v.len() - 1]) + .sum(); + assert_answers_equal( + &answers(&plan, &samples, &deployment, "sum"), + &BTreeMap::from([(String::new(), expected)]), + ); +} + +/// `sum_over_time(m[5m])` (exact) with raw data kept: each series' sum of +/// its 300 samples in (T − 5 min, T]. +#[test] +fn an_exact_range_sum_per_series_equals_the_window_sums() { + let deployment = DeploymentCapabilities::default(); + let plan = plan( + "sum_over_time(m[5m])", + AccuracyTarget::exact(), + SERIES as u64, + &deployment, + ); + let samples = samples(); + let expected = window(&samples, 300_000, 0) + .into_iter() + .map(|(series, values)| (series, values.iter().sum())) + .collect(); + assert_answers_equal(&answers(&plan, &samples, &deployment, "value"), &expected); +} + +/// `sum(sum_over_time(m[5m]))` (exact) with no raw data kept: a chain of +/// 60 s windows of exact sums built at ingestion time, 5 windows merged +/// at query time; the sum of every sample in (T − 5 min, T]. +#[test] +fn an_exact_sum_over_chain_windows_equals_the_sum_of_the_window() { + let deployment = no_raw_data(); + let plan = plan( + "sum(sum_over_time(m[5m]))", + AccuracyTarget::exact(), + SERIES as u64, + &deployment, + ); + assert!( + operator_kinds(&plan).contains(&"SummaryTumblingWindowChain"), + "{:?}", + operator_kinds(&plan) + ); + let samples = samples(); + let expected: f64 = window(&samples, 300_000, 0).values().flatten().sum(); + assert_answers_equal( + &answers(&plan, &samples, &deployment, "sum"), + &BTreeMap::from([(String::new(), expected)]), + ); +} + +/// `sum(sum_over_time(m[5m] offset 10m))` (exact) with no raw data kept: +/// the 5 windows ending 10 min before T, read through a `TimeShift` over +/// the chain; the sum of every sample in (T − 15 min, T − 10 min]. +#[test] +fn an_exact_sum_over_shifted_chain_windows_equals_the_shifted_window_sum() { + let deployment = no_raw_data(); + let plan = plan( + "sum(sum_over_time(m[5m] offset 10m))", + AccuracyTarget::exact(), + SERIES as u64, + &deployment, + ); + let shape = operator_kinds(&plan); + assert!( + shape.contains(&"SummaryTumblingWindowChain") && shape.contains(&"TimeShift"), + "{shape:?}" + ); + let samples = samples(); + let expected: f64 = window(&samples, 300_000, 600_000).values().flatten().sum(); + assert_answers_equal( + &answers(&plan, &samples, &deployment, "sum"), + &BTreeMap::from([(String::new(), expected)]), + ); +} + +/// `quantile_over_time(0.9, m[5m])` within rank error ε = 0.05 per value +/// at δ = 0.01, with no raw data kept, declared over 10,000,000 series so +/// that a sketch costs less than exact state: a sketch chain answers each +/// of the 200 series. A value is off when the fraction of the series' +/// 300 window samples at or below it is outside [0.9 − ε, 0.9 + ε]; at +/// most δ × 200 = 2 values may be off. +#[test] +fn an_approximate_quantile_is_within_its_rank_error_for_all_but_delta_of_the_series() { + let epsilon = 0.05; + let delta = 0.01; + let deployment = no_raw_data(); + let plan = plan( + "quantile_over_time(0.9, m[5m])", + AccuracyTarget::max_error(epsilon) + .failure_probability(delta) + .accuracy_metric(AccuracyMetric::Rank) + .accuracy_scope(AccuracyScope::PerValue), + 10_000_000, + &deployment, + ); + assert!( + plan.nodes.iter().any(|node| node.parameters.is_some()), + "the plan builds no sketch: {:?}", + operator_kinds(&plan) + ); + let samples = samples(); + let windows = window(&samples, 300_000, 0); + let answers = answers(&plan, &samples, &deployment, "value"); + assert_eq!(answers.len(), windows.len(), "one answer per series"); + let off = windows + .iter() + .filter(|(series, values)| { + let answer = answers[*series]; + let at_or_below = values.iter().filter(|v| **v <= answer).count() as f64; + let below = values.iter().filter(|v| **v < answer).count() as f64; + let n = values.len() as f64; + // A rank within ε of 0.9 n exists among the answer's ties. + below / n > 0.9 + epsilon || at_or_below / n < 0.9 - epsilon + }) + .count(); + assert!( + off as f64 <= delta * windows.len() as f64, + "{off} of {} values are off by more than rank error {epsilon}", + windows.len() + ); +} + +/// `max_over_time(m[5m])` (exact) with no raw data kept: per series, the +/// largest of its 300 samples in (T − 5 min, T], from exact max states of +/// 60 s windows. +#[test] +fn an_exact_max_over_chain_windows_equals_the_window_maximum() { + let deployment = no_raw_data(); + let plan = plan( + "max_over_time(m[5m])", + AccuracyTarget::exact(), + SERIES as u64, + &deployment, + ); + assert!( + operator_kinds(&plan).contains(&"SummaryTumblingWindowChain"), + "{:?}", + operator_kinds(&plan) + ); + let samples = samples(); + let expected = window(&samples, 300_000, 0) + .into_iter() + .map(|(series, values)| (series, values.into_iter().fold(f64::MIN, f64::max))) + .collect(); + assert_answers_equal(&answers(&plan, &samples, &deployment, "value"), &expected); +} + +/// `quantile_over_time(0.9, m[5m])` within relative error ε = 0.05 per +/// value at δ = 0.01, no raw data kept, declared over 10,000,000 series: a +/// sketch with a relative guarantee answers each of the 200 series. A +/// value is off when it is outside [(1 − ε) x_lo, (1 + ε) x_hi], x_lo and +/// x_hi the window's sorted samples at ranks ⌊0.9 (n − 1)⌋ and ⌈0.9 (n − +/// 1)⌉ (PromQL interpolates between them); at most δ × 200 = 2 values may +/// be off. +#[test] +fn an_approximate_quantile_is_within_its_relative_error_for_all_but_delta_of_the_series() { + let epsilon = 0.05; + let delta = 0.01; + let deployment = no_raw_data(); + let plan = plan( + "quantile_over_time(0.9, m[5m])", + AccuracyTarget::max_error(epsilon) + .failure_probability(delta) + .accuracy_metric(AccuracyMetric::Relative) + .accuracy_scope(AccuracyScope::PerValue), + 10_000_000, + &deployment, + ); + assert!( + plan.nodes.iter().any(|node| node.parameters.is_some()), + "the plan builds no sketch: {:?}", + operator_kinds(&plan) + ); + let samples = samples(); + let windows = window(&samples, 300_000, 0); + let answers = answers(&plan, &samples, &deployment, "value"); + assert_eq!(answers.len(), windows.len(), "one answer per series"); + let off = windows + .iter() + .filter(|(series, values)| { + let mut sorted = (*values).clone(); + sorted.sort_by(f64::total_cmp); + let position = 0.9 * (sorted.len() - 1) as f64; + let (low, high) = ( + sorted[position.floor() as usize], + sorted[position.ceil() as usize], + ); + let answer = answers[*series]; + answer < (1.0 - epsilon) * low || answer > (1.0 + epsilon) * high + }) + .count(); + assert!( + off as f64 <= delta * windows.len() as f64, + "{off} of {} values are off by more than relative error {epsilon}", + windows.len() + ); +} diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index 255653cb8..a8c35fd75 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -224,6 +224,7 @@ fn tumbling_window_state(maintenance: &CompiledPhysicalDAG, tumbling_window: i64 window_start_ms: tumbling_window * TUMBLING_WINDOW_WIDTH_MS, window_end_ms: (tumbling_window + 1) * TUMBLING_WINDOW_WIDTH_MS, }, + Limits::default(), ); state[0][0].clone() } @@ -322,7 +323,7 @@ fn five_tumbling_windows_roundtrip_and_shared_merge_runs_once() { .err() .unwrap(); assert!(missing.to_string().contains("no rows tiling"), "{missing}"); - let result = physical_common::execute(&compiled, bind(5), scope); + let result = physical_common::execute(&compiled, bind(5), scope, Limits::default()); let kll_state = summary(&result[0][0]); let kll = kll_state .as_any() @@ -524,9 +525,10 @@ fn a_store_keyed_by_definition_and_time_range_answers_the_tumbling_windows() { ); // View `i` holds the tumbling window ending `i` minutes before the evaluation time. let by_node = store_at(&fixture.views, 300_000, |i| tumbling_windows[4 - i].clone()); - let by_node: Vec>> = physical_common::execute(&compiled, by_node, scope) - .iter() - .map(|batches| batches.iter().flat_map(Batch::to_rows).collect()) - .collect(); + let by_node: Vec>> = + physical_common::execute(&compiled, by_node, scope, Limits::default()) + .iter() + .map(|batches| batches.iter().flat_map(Batch::to_rows).collect()) + .collect(); assert_eq!(format!("{by_definition:?}"), format!("{by_node:?}")); } diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index 5ffba1f78..92e28958f 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -9,7 +9,9 @@ mod accuracy_at_query_roots; mod aggregate; mod binary_op; mod contracts; +mod cost_oracle; mod crate_graph; +mod execution_oracle; mod filtered_aggregates; mod finer_aggregate_reuse; mod frontend_timestamps; diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 998ee0ea0..40ce6cc29 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -7,20 +7,19 @@ use asap_executor::{ use futures::{executor::block_on, StreamExt}; use std::sync::Arc; -/// Run `plan` against `store` and collect each root's batches. +/// Run `plan` against `store` within `limits` and collect each root's +/// batches. #[allow(dead_code)] pub fn execute( plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope, + limits: Limits, ) -> Vec> { let dag = plan.instantiate(Arc::new(store)).unwrap(); block_on(async { let streams = dag - .execute( - plan.roots(), - RunContext::new(scope, Limits::default()).unwrap(), - ) + .execute(plan.roots(), RunContext::new(scope, limits).unwrap()) .unwrap(); futures::future::join_all(streams.into_iter().map(|mut stream| async move { let mut batches = Vec::new(); diff --git a/docs/develop_docs/testing.md b/docs/develop_docs/testing.md index 9f9220be7..b4acd9cc2 100644 --- a/docs/develop_docs/testing.md +++ b/docs/develop_docs/testing.md @@ -15,7 +15,7 @@ planner's own costing. |---|---|---|---| | Rule unit tests | one rule over a minimal memo: the candidates, edges or prune causes it adds | a one- or two-query workload planned up to the stage under test through the production frontends and Stage 0 | each stage crate's tests | | Contracts | invariants of every stage's output (below) | every workload of the corpus registry | `crates/integration-tests/tests/contracts.rs` | -| Oracles | the planner's cost or answers against a computation that does not use the planner's costing | canonical shapes and small synthetic data sets | `crates/integration-tests` | +| Oracles | the planner's cost or answers against a computation that does not use the planner's costing | canonical shapes and small synthetic data sets | `crates/integration-tests/tests/{cost_oracle,execution_oracle}.rs` | | Metamorphic relations | how the cost of two related workloads must compare | `planner_fuzz` with fixed seeds | `crates/devtools/tests/planner_fuzz.rs`, the `planner_fuzz` bin | | Regressions | one bug stays fixed | one minimal workload per issue | next to the code the bug was in, or `crates/integration-tests` | | Snapshots | the full selected plan and its cost do not change unexplained | the corpus registry | `crates/devtools/tests/corpus_snapshots.rs`, [planner snapshots](planner-snapshots.md) | @@ -116,6 +116,24 @@ The canonical cost shapes are a `Scan`, a chain built at ingestion time, a offset range query, o11y q21) and a `SummaryMerge` of selected windows. A hand-computed test states each count it uses in its doc comment (for q21: 61 evaluations per 60 s, 5 windows of 1,000 states selected per evaluation). +It costs the selected plan with cost inputs that charge only CPU +operations, or only retained bytes, so each node's expected cost is a count +times one unit; the per-operation work (one operation per row, state or +accumulator) is the cost model's and is stated in the test, the counts are +what the oracle checks. + +The execution oracle runs the plan as a runtime does: each materialized +view (a chain) is refreshed once per window its retention keeps, in an +ingestion run over that window's samples, then the query run reads them. +Its synthetic data is 200 series of one gauge, one sample per second for +30 minutes, each value uniform in [0, 100) from a fixed seed plus 10 times +the minute it falls in, so a plan that reads the wrong windows moves its +answer far outside ε; an approximate query is checked over all 200 values +(at most δ × 200 may be off), and two answers for one series fail. The +query run is bound raw data only when the deployment keeps it, so a plan +that scans raw data on a deployment without it fails. A +query shape the executor cannot run yet is ignored with the executor +issue that adds it, not reshaped. ### Metamorphic relations @@ -252,13 +270,13 @@ Tokens: | sum | S3+X `planner/tests/runtime_install.rs::an_instant_selector_reads_the_latest_sample_per_series` | S3+X `integration-tests/tests/filtered_aggregates.rs::exact_filtered_sum_is_null_for_groups_without_matches` | L `frontend-metricsql/tests/lowering.rs::selector_range_aggregate_and_call_share_the_canonical_shape`; exec gap | | avg | S3 `integration-tests/tests/joint_search_equivalence.rs::composed_quantile_and_sum_outputs_are_validated_together`; Xd (lowered) `integration-tests/tests/per_series_exact_execution.rs::exact_per_series_range_functions_execute` (avg_over_time) | S3+X `integration-tests/tests/stage1_logical_asap_optimization.rs::sql_avg_from_sum_and_count_executes_as_avg` | gap (only a test that rejects `avg(foo, bar)`) | | count | S3 `planner/tests/e2e_plan.rs::each_plan_carries_its_selected_root` (count_over_time); Xd (lowered) `integration-tests/tests/per_series_exact_execution.rs::exact_per_series_range_functions_execute` | S3+X `integration-tests/tests/filtered_aggregates.rs::exact_filtered_count_keeps_groups_without_matches` | L `frontend-metricsql/tests/lowering_semantics.rs::count_counts_series`; exec gap | -| min / max | S3 `devtools/tests/asap_planner.rs::a_counter_function_over_a_declared_gauge_plans_with_a_warning` (max_over_time), `devtools/tests/planner_fuzz.rs::threshold_witness_costs_over_the_result` (min by); Xd (lowered) `integration-tests/tests/per_series_exact_execution.rs::exact_per_series_range_functions_execute` | S3 `integration-tests/tests/independent_plan_seed.rs::percentile_and_max_over_one_scan_cost_no_more_than_separate_plans`; Xd (hand-built) `executor/tests/physical_semantics.rs::global_extrema_bind_with_planner_derived_schema` | gap | -| quantile | S3 `planner/tests/runtime_install.rs::the_plan_carries_stage3s_predicted_usage`; Xd (lowered) `integration-tests/tests/per_series_exact_execution.rs::quantile_bounds_follow_promql` (exact only; `planner/tests/columnar_batches.rs` runs a KLL build but checks no answer) | S3+X `integration-tests/tests/filtered_aggregates.rs::kll_quantile_of_all_null_group_is_null` (KLL plan; answers on trivial groups only, no ε check) | L `frontend-metricsql/tests/lowering.rs::supported_parameterized_functions_require_their_exact_arity`; exec gap | +| min / max | S3 `devtools/tests/asap_planner.rs::a_counter_function_over_a_declared_gauge_plans_with_a_warning` (max_over_time), `devtools/tests/planner_fuzz.rs::threshold_witness_costs_over_the_result` (min by); Xd (lowered) `integration-tests/tests/per_series_exact_execution.rs::exact_per_series_range_functions_execute`; S3+X `integration-tests/tests/execution_oracle.rs::an_exact_max_over_chain_windows_equals_the_window_maximum` (max_over_time over a chain) | S3 `integration-tests/tests/independent_plan_seed.rs::percentile_and_max_over_one_scan_cost_no_more_than_separate_plans`; Xd (hand-built) `executor/tests/physical_semantics.rs::global_extrema_bind_with_planner_derived_schema` | gap | +| quantile | S3 `planner/tests/runtime_install.rs::the_plan_carries_stage3s_predicted_usage`; Xd (lowered) `integration-tests/tests/per_series_exact_execution.rs::quantile_bounds_follow_promql` (exact only; `planner/tests/columnar_batches.rs` runs a KLL build but checks no answer); S3+X `integration-tests/tests/execution_oracle.rs::an_approximate_quantile_is_within_its_rank_error_for_all_but_delta_of_the_series` (sketch chain, rank ε; `::an_approximate_quantile_is_within_its_relative_error_for_all_but_delta_of_the_series`, relative ε) | S3+X `integration-tests/tests/filtered_aggregates.rs::kll_quantile_of_all_null_group_is_null` (KLL plan; answers on trivial groups only, no ε check) | L `frontend-metricsql/tests/lowering.rs::supported_parameterized_functions_require_their_exact_arity`; exec gap | | topk / bottomk | topk: S3 `devtools/tests/asap_planner.rs::example1_document_validates_its_stage_dags`; Xd (lowered) `executor/tests/current_series_heap.rs::population_fixture_compiles_with_dynamic_identity`. Heap-sketch answers are checked only in `integration-tests/tests/promql_to_post_asap.rs::planner_heap_topk_reference_execution_matches_ground_truth` [ign], by an evaluator inside the test, not the executor. bottomk: L only `devtools/tests/cross_language.rs::ascending_count_ranked_topk_stays_generic_in_both_languages` | ORDER BY value … LIMIT: S3 `integration-tests/tests/sql_to_post_asap.rs::sql_relational_parents_retain_summary_bound_aggregate`. Top-k by count (heavy hitter) stops at Pass 1: `integration-tests/tests/per_computation_candidates_sql.rs::topk_heap_over_an_int64_key_counts_raw_rows`. Xd (hand-built) `executor/tests/physical_semantics.rs::limit_offset_fetch_matrix` | gap | | distinct count | L `frontend-promql/tests/promql_lowering.rs::distinct_over_time_lowers_to_cardinality_over_nested_windows`; S3 gap; exec gap | S3+X `planner/tests/sql_table_scans.rs::ingestion_time_tumbling_windows_built_from_table_rows_answer_exactly` | gap | | rate / increase | S3 `devtools/tests/asap_planner.rs::example1_document_validates_its_stage_dags` (rate), `integration-tests/tests/promql_to_post_asap.rs::value_ranked_topk_over_binary_ratio_finalizes_both_summary_operands` (increase); Xd (hand-built) `executor/tests/promql_values.rs::recovered_exact_counter_uses_window_and_omits_insufficient_samples` | n/a | L `frontend-metricsql/tests/lowering.rs::selector_range_aggregate_and_call_share_the_canonical_shape`; exec gap | | binary op | S3 `integration-tests/tests/promql_to_post_asap.rs::promql_binary_arithmetic_preserves_nested_structure_and_vector_matching`; Xd (hand-built) `executor/tests/promql_binary.rs::compiled_binary_matches_series_and_preserves_checked_division` | S3 `devtools/tests/asap_planner.rs::example2_plans_the_sql_workload` (entropy and L2 arithmetic); Xd (lowered) `integration-tests/tests/sql_frequency.rs::recognized_sql_l2_executes_exactly` | gap (`sum(up) + sum(down)` appears only as an error case) | -| offset | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query`; Xd (hand-built) `executor/tests/time_series_window.rs::a_shifted_range_window_ends_its_offset_before_the_evaluation` | n/a (shifted `ts` window predicate: gap) | gap | +| offset | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query`; Xd (hand-built) `executor/tests/time_series_window.rs::a_shifted_range_window_ends_its_offset_before_the_evaluation`; S3+X `integration-tests/tests/execution_oracle.rs::an_exact_sum_over_shifted_chain_windows_equals_the_shifted_window_sum` | n/a (shifted `ts` window predicate: gap) | gap | | `@` modifier | L `integration-tests/tests/nested.rs::q40_at_modifier_absolute`. Stage 2 unit test on a hand-built memo: `stage2-physical-optimization/src/catalog/tests.rs::an_at_t_subtree_is_offered_ingestion_time`. S3 gap. Xd (hand-built) `executor/tests/tumbling_window_chain.rs::a_time_shift_at_end_evaluates_its_input_at_the_requests_end` | n/a | gap | | subquery | S3 only [ign]: `devtools/tests/corpus_snapshots/o11y_bench/q05.json` (and q06, q11) via `devtools/tests/corpus_snapshots.rs::corpus_plans_match_snapshots`. L `integration-tests/tests/nested.rs::q27_nested_subquery_prometheus_docs_example`. Xd (hand-built; memory limit only) `executor/tests/series_window_memory_limit.rs::a_long_subquery_over_many_series_fails_near_the_memory_limit` | derived table: S3 `integration-tests/tests/sql_to_post_asap.rs::sql_relational_parents_retain_summary_bound_aggregate`; Xd (lowered) `integration-tests/tests/sql_frequency.rs::recognized_sql_entropy_executes_in_nats` | gap | @@ -274,16 +292,16 @@ PromQL: | operator | range, exact | range, ε | repeating, exact | repeating, ε | |---|---|---|---|---| -| sum | gap | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query` | S3 `integration-tests/tests/sharing_issue_regressions.rs::nested_exact_sum_reuses_newest_tumbling_windows` | S3+X `planner/tests/runtime_install.rs::an_instant_selector_reads_the_latest_sample_per_series` (rank 0.01) | +| sum | gap | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query` | S3+X `integration-tests/tests/execution_oracle.rs::an_exact_sum_over_chain_windows_equals_the_sum_of_the_window` | S3+X `planner/tests/runtime_install.rs::an_instant_selector_reads_the_latest_sample_per_series` (rank 0.01) | | avg | gap | gap | S3 [ign] `integration-tests/tests/sharing_issue_regressions.rs::exact_coarse_sum_reuses_fine_sum` (avg_over_time) | S3 `integration-tests/tests/joint_search_equivalence.rs::composed_quantile_and_sum_outputs_are_validated_together` | | count | gap | gap | S3 `devtools/tests/planner_fuzz.rs::exact_rollup_witness_costs_for_a_grouped_count` | S3 `planner/tests/e2e_plan.rs::each_plan_carries_its_selected_root` (count_over_time) | -| min / max | gap | S3 [ign] `devtools/tests/corpus_snapshots.rs::corpus_plans_match_snapshots` (q11, max_over_time) | S3 `devtools/tests/planner_fuzz.rs::threshold_witness_costs_over_the_result` (min by over max_over_time) | gap | -| quantile | gap | S3 [ign] `devtools/tests/corpus_snapshots.rs::corpus_plans_match_snapshots` (q02, `histogram_quantile` only) | gap | S3 `integration-tests/tests/accuracy_at_query_roots.rs::a_whole_result_failure_probability_is_divided_over_the_values` | +| min / max | gap | S3 [ign] `devtools/tests/corpus_snapshots.rs::corpus_plans_match_snapshots` (q11, max_over_time) | S3+X `integration-tests/tests/execution_oracle.rs::an_exact_max_over_chain_windows_equals_the_window_maximum` | gap | +| quantile | gap | S3 [ign] `devtools/tests/corpus_snapshots.rs::corpus_plans_match_snapshots` (q02, `histogram_quantile` only) | gap | S3+X `integration-tests/tests/execution_oracle.rs::an_approximate_quantile_is_within_its_rank_error_for_all_but_delta_of_the_series` | | topk | gap | gap | S3 `integration-tests/tests/sharing_issue_regressions.rs::duplicate_exact_topk_shares_work` | S3 `integration-tests/tests/joint_search_equivalence.rs::fused_top_k_preserves_its_independently_requested_inner_sum` | | distinct count | gap | gap | gap | gap | | rate / increase | gap | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query` | S3 `devtools/tests/asap_planner_acceptance.rs::stage3_selects_candidates_stage2_lists` | S3 `planner/tests/summary_sharing.rs::identical_ungrouped_queries_share_their_producers` | | binary op | gap | S3 [ign] `devtools/tests/corpus_snapshots.rs::corpus_plans_match_snapshots` (q19, increase ratio) | S3 `integration-tests/tests/production_rollup_workloads.rs::production_binary_queries_use_joint_prices_with_mixed_schedules` | S3 `integration-tests/tests/accuracy_at_query_roots.rs::each_query_meets_its_target_at_its_root` (quantile ratio) | -| offset | L `frontend-promql/src/lib.rs::tests::at_start_and_at_end_are_lifted_above_the_selector_they_anchor` | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query` | gap | gap | +| offset | L `frontend-promql/src/lib.rs::tests::at_start_and_at_end_are_lifted_above_the_selector_they_anchor` | S3 `integration-tests/tests/milp_objective.rs::the_milp_objective_is_the_plan_cost_on_an_offset_range_query` | S3+X `integration-tests/tests/execution_oracle.rs::an_exact_sum_over_shifted_chain_windows_equals_the_shifted_window_sum` | gap | SQL has no range queries (no test sets `range_evaluation`, and the SQL frontend does not read it): @@ -298,8 +316,8 @@ frontend does not read it): | topk (ORDER BY … LIMIT) | gap | gap | | distinct count | gap | S3+X `planner/tests/sql_table_scans.rs::ingestion_time_tumbling_windows_built_from_table_rows_answer_exactly` | -No range query has an exact target or is executed; only one PromQL -cell and two SQL ones reach S3+X. +No range query has an exact target or is executed (the q21 range query's +cost is hand-computed: `integration-tests/tests/cost_oracle.rs::a_time_range_over_a_shifted_chain_merges_every_selected_window`). ### Time shape @@ -392,9 +410,9 @@ The 11 ignored tests: ### Notable gaps 1. MetricSQL is never planned. No test sends a `metricsql` workload or `FrontendInput::Metricsql` to `e2e_plan` or `plan_installable`, even though the facade supports it. Only sum, count, quantile and rate are lowered. -2. Few planner-selected plans are executed with answers checked (`X`). PromQL has only `sum` (`runtime_install`). SQL has sum, count, avg, quantile (KLL) and distinct count. PromQL quantile, topk, rate, binary operations, offset, `@` and subquery are never executed from a Stage 3 plan. -3. No PromQL plan built on a sketch (KLL, DDSketch, CMS heap) has its answers checked by the executor. The heap check is ignored and uses an evaluator written inside the test. `columnar_batches` runs a KLL build without checking an answer. -4. Range queries (`range_evaluation`) have one non-ignored test, which checks cost only (`milp_objective`). There is no SQL range query, and no range query is ever executed. +2. Few planner-selected plans are executed with answers checked (`X`). PromQL has sum, sum_over_time, max_over_time, quantile_over_time (a sketch chain) and offset over a chain, at one evaluation time (`runtime_install`, `execution_oracle`). SQL has sum, count, avg, quantile (KLL) and distinct count. PromQL topk, rate, binary operations, `@` and subquery are never executed from a Stage 3 plan. +3. The only PromQL sketch plans whose answers the executor checks are the quantile sketch chains of `execution_oracle` (rank and relative ε). No CMS heap or top-k sketch plan is executed: the heap check is ignored and uses an evaluator written inside the test. `columnar_batches` runs a KLL build without checking an answer. +4. Range queries (`range_evaluation`) have two non-ignored tests, which check cost only (`milp_objective` and the q21 hand-computed cost in `cost_oracle`). There is no SQL range query, and no range query is ever executed. 5. Out-of-order data appears in a single Stage 2 unit test on a hand-built memo. No test in either language sets `max_out_of_order_ms` through the pipeline or executes late samples. 6. The `@` modifier is not planned past Stage 2 from query text. Subqueries reach Stage 3 only in the ignored corpus snapshots. Neither is executed except by operator-level tests (`Xd`, hand-built). 7. The corpus snapshots (14 `o11y_bench` queries: subquery, offset, histogram_quantile, range_evaluation) and the sketch plan snapshots (10 workloads) are `#[ignore]` and do not run by default.