diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index 2a06f2b5c..87b5fab80 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -3,6 +3,7 @@ //! One file: `cargo test -p asap-integration-tests --test integration ::`. mod physical_common; +mod plan_enumeration; mod planner_layering_common; mod planning_common; @@ -32,5 +33,6 @@ mod schema; mod sql_cardinality; mod sql_frequency; mod sql_to_post_asap; +mod stage3_installable_optimal_selection; mod stage3_objective; mod time_range; diff --git a/crates/integration-tests/tests/plan_enumeration/mod.rs b/crates/integration-tests/tests/plan_enumeration/mod.rs new file mode 100644 index 000000000..ac0e4c673 --- /dev/null +++ b/crates/integration-tests/tests/plan_enumeration/mod.rs @@ -0,0 +1,603 @@ +//! A test oracle for Stage 3: every plan of a physical memo (Stage 2's +//! output), built and costed without Stage 3's selectable executions, cost +//! coefficients or MILP, each with `asap_cost::plan_cost` and checked +//! against the planner's install rules (`PhysicalASAPDAG::materialized_views` +//! and `PhysicalASAPDAG::raw_data_retention`). +//! +//! A plan is one execution per (execution context, physical candidate) it +//! uses (concepts doc, section 5), each input of an execution read from one +//! producer execution over an admissible physical edge, each query output +//! answered by one execution of its group: +//! +//! - a query output is answered in the query's evaluation context (its grid, +//! `QueryWorkloadEntry::evaluation_grid_ms`, at phase 0) by a candidate a +//! query-time consumer may read (`producer_satisfies_physical_requirements`); +//! - a producer is read in a context only where it delivers on it +//! (`PhysicalASAPMemoCandidates::delivers_on`); +//! - a producer materialized across evaluations runs once, in +//! `Materialized(v)`; another ingestion-time producer in the one ingestion +//! context `ExecutionContext::EVERY_ARRIVING_ROW`; any other in its +//! consumer's context; +//! - the inputs of an ingestion-time candidate materialized across +//! evaluations run on the ingestion pipeline of its refresh window +//! (`RefreshWindow::computed_from` over the candidates); of any other +//! candidate, in its own context; +//! - a query-time result that is not materialized has one reader per +//! context (`PhysicalProperties::has_one_reader_per_context`); +//! - the plan is acyclic, and a query reads no candidate it must not read +//! (`LogicalASAPMemoCandidates::query_must_not_read`) anywhere below its +//! output. +//! +//! Each node retaining rows (`PhysicalASAPDAGNode::retains_rows`) retains +//! the least its plan's readers need: `asap_cost::read_back_ms` over the +//! built plan, each reader's range-evaluation span the longest of the +//! queries reaching it in the plan, and at least the span of a query it +//! answers directly; `None` when read unbounded. +//! +//! Not modeled: a `TimeShift` anchored with `@ end()` or `@ start()` (its +//! inputs run once per request, in a context of its own), and accuracy +//! constraint (K) (Stage 3 does not have it yet). A memo with an anchored +//! shift is refused ([`EnumerationError::AnchoredShift`]). + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use asap_cost::CostInputs; +use asap_types::ir::properties::{DataPrimitive, ExecutionDataState, ExecutionTiming}; +use asap_types::ir::QueryRoot; +use asap_types::plan::materialized_view::{InstallableDagError, RefreshWindow}; +use asap_types::plan::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, +}; +use asap_types::plan::view_coverage; +use asap_types::stages::{ + producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, MemoGroupId, + PhysicalASAPMemoCandidates, PhysicalCandidateId, +}; +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; + +/// One execution: a candidate in a context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct Execution { + context: ExecutionContext, + candidate: PhysicalCandidateId, +} + +/// A partial plan while it is enumerated. +#[derive(Debug, Clone, Default)] +struct PartialPlan { + executions: Vec, + position: HashMap, + /// Per execution, per input, the producer execution chosen. + producers: Vec>>, + /// Per execution, how many edges and query outputs read it. + reads: Vec, + /// Per query output (`Memo::output_groups` order), the execution + /// answering it. + answers: Vec>, + /// The choices left: query outputs, then inputs of executions added. + pending: Vec, +} + +#[derive(Debug, Clone, Copy)] +enum Slot { + Output(usize), + Input { consumer: usize, input: usize }, +} + +/// One enumerated plan, with its `plan_cost` per second (cost model 1). +#[derive(Debug, Clone)] +pub struct EnumeratedPlan { + pub plan: PhysicalASAPDAG, + pub cost_per_second: f64, + /// The first install rule it fails, `None` when it installs. + pub install_error: Option, +} + +/// Every plan of a physical memo, summarized. +#[derive(Debug, Default)] +pub struct Enumeration { + /// Complete choices enumerated. + pub choices: usize, + /// Of them, plans refused before costing: a cycle, an exact-only read + /// of a sketch, or an invalid DAG. + pub refused: usize, + /// Of them, plans `plan_cost` could not cost, and the first error. + pub not_costable: usize, + pub first_cost_error: Option, + /// The cheapest plan that installs. + pub cheapest_installable: Option, + /// The cheapest plan, installable or not. + pub cheapest: Option, + /// How many costed plans install, and how many do not. + pub installable: usize, + pub not_installable: usize, + /// The [`plan_signature`] of every plan built (before costing). + pub plan_signatures: HashSet, +} + +#[derive(Debug, PartialEq)] +pub enum EnumerationError { + /// More than the cap of complete choices: the oracle cannot run. + TooManyPlans(usize), + /// A `TimeShift` anchored with `@ end()` or `@ start()`: not modeled. + AnchoredShift(PhysicalCandidateId), +} + +impl std::fmt::Display for EnumerationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooManyPlans(cap) => write!(f, "more than {cap} complete choices"), + Self::AnchoredShift(v) => write!(f, "candidate v{} is an anchored TimeShift", v.0), + } + } +} + +/// Every plan of `physical` for `entries` (query `q` is `entries[q]`), each +/// costed with `plan_cost` under `cost_inputs`, at most `max_choices` +/// complete choices. +pub fn enumerate_plans( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + data: &DataWorkload, + cost_inputs: &CostInputs, + max_choices: usize, +) -> Result { + if let Some(v) = (0..physical.physical_candidates.len()) + .map(PhysicalCandidateId) + .find(|&v| physical.operator(v).request_anchor().is_some()) + { + return Err(EnumerationError::AnchoredShift(v)); + } + let outputs = physical.logical_candidates.memo.output_groups(); + let start = PartialPlan { + answers: vec![None; outputs.len()], + pending: (0..outputs.len()).rev().map(Slot::Output).collect(), + ..PartialPlan::default() + }; + let mut enumerator = Enumerator { + physical, + entries, + data, + cost_inputs, + outputs: &outputs, + max_choices, + found: Enumeration::default(), + }; + enumerator.extend(start)?; + Ok(enumerator.found) +} + +struct Enumerator<'a> { + physical: &'a PhysicalASAPMemoCandidates, + entries: &'a [QueryWorkloadEntry], + data: &'a DataWorkload, + cost_inputs: &'a CostInputs, + outputs: &'a [(usize, MemoGroupId)], + max_choices: usize, + found: Enumeration, +} + +impl Enumerator<'_> { + fn extend(&mut self, mut partial: PartialPlan) -> Result<(), EnumerationError> { + let Some(slot) = partial.pending.pop() else { + self.found.choices += 1; + if self.found.choices > self.max_choices { + return Err(EnumerationError::TooManyPlans(self.max_choices)); + } + self.complete(&partial); + return Ok(()); + }; + for (context, candidate) in self.options(&partial, slot) { + let mut next = partial.clone(); + let execution = Execution { context, candidate }; + let index = match next.position.get(&execution) { + Some(&index) => { + let one_reader = self.physical.physical_candidates[candidate.0] + .physical_properties + .has_one_reader_per_context(); + if one_reader && next.reads[index] >= 1 { + continue; + } + index + } + None => { + let index = next.executions.len(); + next.executions.push(execution); + next.position.insert(execution, index); + let inputs = self.physical.input_count(candidate); + next.producers.push(vec![None; inputs]); + next.reads.push(0); + for input in (0..inputs).rev() { + next.pending.push(Slot::Input { + consumer: index, + input, + }); + } + index + } + }; + next.reads[index] += 1; + match slot { + Slot::Output(output) => next.answers[output] = Some(index), + Slot::Input { consumer, input } => next.producers[consumer][input] = Some(index), + } + self.extend(next)?; + } + Ok(()) + } + + /// The executions (context, candidate) that may fill `slot`. + fn options( + &self, + partial: &PartialPlan, + slot: Slot, + ) -> Vec<(ExecutionContext, PhysicalCandidateId)> { + let physical = self.physical; + match slot { + Slot::Output(output) => { + let (query, group) = self.outputs[output]; + let context = ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms: self.entries[query].evaluation_grid_ms(), + phase_ms: 0, + }, + }; + (0..physical.physical_candidates.len()) + .map(PhysicalCandidateId) + .filter(|&v| { + let candidate = &physical.physical_candidates[v.0]; + candidate.logical_candidate_id.group == group + && producer_satisfies_physical_requirements( + candidate, + ExecutionTiming::QueryTime, + ) + }) + .filter_map(|v| self.execution_reading(context, v)) + .collect() + } + Slot::Input { consumer, input } => { + let consumer = partial.executions[consumer]; + let context = self.input_context(consumer); + physical + .admissible_edges_into(consumer.candidate, input) + .iter() + .map(|&edge| physical.edge(edge).producer_candidate_id) + .collect::>() + .into_iter() + .filter_map(|u| self.execution_reading(context, u)) + .collect() + } + } + } + + /// The execution of `u` a consumer whose inputs run in `context` reads, + /// when `u` delivers on it. + fn execution_reading( + &self, + context: ExecutionContext, + u: PhysicalCandidateId, + ) -> Option<(ExecutionContext, PhysicalCandidateId)> { + if !self.physical.delivers_on(u, &context) { + return None; + } + let properties = self.physical.physical_candidates[u.0].physical_properties; + let context = if properties.is_across_evaluations() { + ExecutionContext::Materialized(u) + } else if properties.timing() == ExecutionTiming::IngestionTime { + ExecutionContext::EVERY_ARRIVING_ROW + } else { + context + }; + Some((context, u)) + } + + /// The context the inputs of `consumer` run in. + fn input_context(&self, consumer: Execution) -> ExecutionContext { + let properties = + self.physical.physical_candidates[consumer.candidate.0].physical_properties; + if properties.timing() == ExecutionTiming::IngestionTime + && properties.is_across_evaluations() + { + if let Some(window) = RefreshWindow::computed_from(self.physical, consumer.candidate) { + return ExecutionContext::IngestionPipeline(window); + } + } + consumer.context + } + + /// Builds, checks and costs one complete choice. + fn complete(&mut self, partial: &PartialPlan) { + let Some(plan) = self.build(partial) else { + self.found.refused += 1; + return; + }; + self.found.plan_signatures.insert(plan_signature(&plan)); + let cost_per_second = + match asap_cost::plan_cost(&plan, self.entries, self.data, self.cost_inputs) { + Ok((cost, _)) => cost.cost_per_second.expect("cost model 1"), + Err(error) => { + self.found.not_costable += 1; + self.found + .first_cost_error + .get_or_insert_with(|| format!("{error:?}")); + return; + } + }; + let install_error = plan + .materialized_views() + .err() + .or_else(|| plan.raw_data_retention().err()); + let enumerated = EnumeratedPlan { + plan, + cost_per_second, + install_error, + }; + let cheaper = |best: &Option| { + best.as_ref() + .is_none_or(|best| enumerated.cost_per_second < best.cost_per_second) + }; + if enumerated.install_error.is_none() { + self.found.installable += 1; + if cheaper(&self.found.cheapest_installable) { + self.found.cheapest_installable = Some(enumerated.clone()); + } + } else { + self.found.not_installable += 1; + } + if cheaper(&self.found.cheapest) { + self.found.cheapest = Some(enumerated); + } + } + + /// The plan of a complete choice, `None` when it has a cycle, a query + /// reads a candidate it must not read, or the DAG is invalid. + fn build(&self, partial: &PartialPlan) -> Option { + let physical = self.physical; + let count = partial.executions.len(); + let producers = |e: usize| partial.producers[e].iter().map(|p| p.expect("complete")); + // (X): no query reads, below its output, a candidate it must not. + for (output, &(query, _)) in self.outputs.iter().enumerate() { + let mut seen = vec![false; count]; + let mut stack = vec![partial.answers[output].expect("complete")]; + while let Some(e) = stack.pop() { + if std::mem::replace(&mut seen[e], true) { + continue; + } + let logical = physical.physical_candidates[partial.executions[e].candidate.0] + .logical_candidate_id; + if physical + .logical_candidates + .query_must_not_read(query, logical) + { + return None; + } + stack.extend(producers(e)); + } + } + // Producers first (Kahn's order); a cycle leaves executions out. + let mut waiting: Vec = (0..count).map(|e| partial.producers[e].len()).collect(); + let mut consumers: Vec> = vec![Vec::new(); count]; + for e in 0..count { + for p in producers(e) { + consumers[p].push(e); + } + } + let mut ready: Vec = (0..count).rev().filter(|&e| waiting[e] == 0).collect(); + let mut node_of = vec![usize::MAX; count]; + let mut order = Vec::with_capacity(count); + while let Some(e) = ready.pop() { + node_of[e] = order.len(); + order.push(e); + for &c in &consumers[e] { + waiting[c] -= 1; + if waiting[c] == 0 { + ready.push(c); + } + } + } + if order.len() != count { + return None; + } + let nodes: Vec = order + .iter() + .enumerate() + .map(|(id, &e)| { + let inputs: Vec = producers(e).map(|p| node_of[p]).collect(); + plan_node(physical, partial.executions[e].candidate, id, &inputs) + }) + .collect(); + let node_of_output: HashMap<(usize, MemoGroupId), PhysicalASAPNodeId> = self + .outputs + .iter() + .enumerate() + .map(|(output, &key)| (key, node_of[partial.answers[output].expect("complete")])) + .collect(); + let mut roots = Vec::new(); + let mut scalar_roots = Vec::new(); + for (query, memo_query) in physical + .logical_candidates + .memo + .queries() + .iter() + .enumerate() + { + let node = |group: &MemoGroupId| node_of_output[&(query, *group)]; + match &memo_query.root { + QueryRoot::Operator(group) => roots.push(node(group)), + QueryRoot::Scalar(expression) => scalar_roots.push(( + query, + expression.map_operator_refs(&mut |group| node(group)), + )), + } + } + let mut plan = PhysicalASAPDAG::new(nodes, roots, scalar_roots).ok()?; + set_least_retention(&mut plan, self.entries); + plan.validate().ok()?; + Some(plan) + } +} + +/// Candidate `v` as plan node `id` reading `inputs`: its operator, timing, +/// lifetime, θ, its group's schema, and, materialized across evaluations, +/// its group's coverage; no retention. +fn plan_node( + physical: &PhysicalASAPMemoCandidates, + v: PhysicalCandidateId, + id: PhysicalASAPNodeId, + inputs: &[PhysicalASAPNodeId], +) -> PhysicalASAPDAGNode { + let memo = &physical.logical_candidates.memo; + let candidate = &physical.physical_candidates[v.0]; + let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize].group_properties; + let mut inputs = inputs.iter(); + let payload = physical + .operator(v) + .map_children(|_| *inputs.next().expect("one node per input")); + PhysicalASAPDAGNode { + id, + output_state: ExecutionDataState { + timing: candidate.physical_properties.timing(), + primitive: DataPrimitive::of(&payload, group.result_kind), + }, + payload, + lifetime: candidate.physical_properties.lifetime(), + parameters: memo + .candidate(candidate.logical_candidate_id) + .logical_properties + .parameters() + .cloned(), + output_schema: group.schema.clone(), + retention_ms: None, + selected_physical_candidate: Some(v), + view_coverage: candidate + .physical_properties + .is_across_evaluations() + .then(|| view_coverage(memo, candidate.logical_candidate_id.group)) + .flatten(), + } +} + +/// Per node of `plan`, the queries whose walk from their roots reaches it. +pub fn queries_reaching(plan: &PhysicalASAPDAG) -> BTreeMap> { + let mut reaching: BTreeMap> = BTreeMap::new(); + for (query, starts) in query_starts(plan).into_iter().enumerate() { + let mut stack = starts; + while let Some(id) = stack.pop() { + if reaching.entry(id).or_default().insert(query) { + stack.extend( + plan.edges + .iter() + .filter(|e| e.consumer == id) + .map(|e| e.producer), + ); + } + } + } + reaching +} + +/// Per query, the nodes it reads directly: its operator root, or the +/// operators its scalar root reads. +fn query_starts(plan: &PhysicalASAPDAG) -> Vec> { + let queries = plan.roots.len() + plan.scalar_roots.len(); + let mut starts = vec![Vec::new(); queries]; + let scalar: BTreeMap> = plan + .scalar_roots + .iter() + .map(|(query, expression)| { + ( + *query, + expression.operator_refs().into_iter().copied().collect(), + ) + }) + .collect(); + let mut roots = plan.roots.iter(); + for (query, start) in starts.iter_mut().enumerate() { + *start = match scalar.get(&query) { + Some(nodes) => nodes.clone(), + None => vec![*roots.next().expect("one root per operator query")], + }; + } + starts +} + +/// Sets each node of `plan` that retains rows to the least its readers in +/// `plan` need (module documentation). +pub fn set_least_retention(plan: &mut PhysicalASAPDAG, entries: &[QueryWorkloadEntry]) { + let reaching = queries_reaching(plan); + let span_ms = |query: usize| { + entries + .get(query) + .and_then(|e| e.range_evaluation) + .map_or(0, |r| r.span_ms) + }; + let longest_span_ms = |id: PhysicalASAPNodeId| { + reaching + .get(&id) + .into_iter() + .flatten() + .map(|&q| span_ms(q)) + .max() + .unwrap_or(0) + }; + let starts = query_starts(plan); + let retention: Vec<(PhysicalASAPNodeId, Option)> = plan + .nodes + .iter() + .filter(|node| node.retains_rows()) + .map(|node| { + let direct_ms = starts + .iter() + .enumerate() + .filter(|(_, start)| start.contains(&node.id)) + .map(|(query, _)| span_ms(query)) + .max() + .unwrap_or(0); + let read_back = asap_cost::read_back_ms(&*plan, node.id, longest_span_ms); + (node.id, read_back.map(|ms| ms.max(direct_ms))) + }) + .collect(); + for (id, retention_ms) in retention { + if let Some(node) = plan.nodes.iter_mut().find(|node| node.id == id) { + node.retention_ms = retention_ms; + } + } +} + +/// A plan's executions and edges as text: per query start, its node's +/// candidate and, recursively, its inputs' in input order. Two plans with +/// one signature select the same candidates over the same edges. +/// +/// Example: `v7(v3(v0)) | v7(v3(v0))` for two queries answered by one +/// node. +pub fn plan_signature(plan: &PhysicalASAPDAG) -> String { + fn of(plan: &PhysicalASAPDAG, id: PhysicalASAPNodeId) -> String { + let node = plan + .nodes + .iter() + .find(|node| node.id == id) + .expect("a node"); + let inputs: Vec = node + .payload + .children() + .into_iter() + .map(|&input| of(plan, input)) + .collect(); + format!( + "v{}({})", + node.selected_physical_candidate.map_or(usize::MAX, |v| v.0), + inputs.join(",") + ) + } + query_starts(plan) + .iter() + .map(|start| { + start + .iter() + .map(|&id| of(plan, id)) + .collect::>() + .join("+") + }) + .collect::>() + .join(" | ") +} diff --git a/crates/integration-tests/tests/stage3_installable_optimal_selection.rs b/crates/integration-tests/tests/stage3_installable_optimal_selection.rs new file mode 100644 index 000000000..3bf3c3d18 --- /dev/null +++ b/crates/integration-tests/tests/stage3_installable_optimal_selection.rs @@ -0,0 +1,1051 @@ +//! Two contracts of Stage 3's `select_plan`, checked against an oracle that +//! enumerates every plan of the physical memo without Stage 3's cost +//! coefficients or MILP and costs each with `asap_cost::plan_cost` +//! (`plan_enumeration`): +//! +//! - **Installable selection.** Stage 3 never returns a plan that the +//! planner's own install rules refuse (`PhysicalASAPDAG::materialized_views` +//! and `PhysicalASAPDAG::raw_data_retention`). A rule over one node's +//! subtree excludes that node's candidate (`PruneReason::NotInstallable`) +//! and the selection runs again; when no plan installs, Stage 3 returns an +//! error. +//! - **Optimality.** The selected plan's `plan_cost` is the least +//! `plan_cost` of any installable plan, within a relative 1e-6. A cost +//! coefficient wrong only on candidates the MILP did not select is +//! invisible to "objective = `plan_cost` of the selected plan"; it is not +//! invisible here. +//! +//! Expected costs come from the oracle; no expected value is read from +//! Stage 3. + +use asap_cost::{CostInputs, CostModel, ResourceUnitCost}; +use asap_devtools::planning_stages::{ + lower_promql_roots, lower_sql_roots, run_stages_until, sql_catalog, RunUntil, +}; +use asap_planner::{DeploymentCapabilities, PlanningModels, SelectionError}; +use asap_stage3_plan_selection::Selection; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::stages::{Lifetime, PhysicalASAPMemoCandidates, PhysicalCandidateId, PruneReason}; +use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; +use serde_json::{json, Value}; + +use crate::plan_enumeration::{ + enumerate_plans, plan_signature, queries_reaching, set_least_retention, Enumeration, +}; + +/// The relative tolerance of a cost comparison: HiGHS solves at gap 0, so +/// only its feasibility tolerance and the objective's scaling separate the +/// optimum from the cheapest plan. +const RELATIVE_TOLERANCE: f64 = 1e-6; + +/// At most this many complete choices per workload; above it the oracle +/// cannot run and the test fails saying so. +const MAX_CHOICES: usize = 400_000; + +/// One workload of the contracts. +#[derive(Clone)] +struct Workload { + name: &'static str, + /// The planning workload (queries and data). + planning: PlanningWorkload, + /// SQL table declarations, `None` for PromQL. + sql_tables: Option>, + raw_data_retained: bool, + ignore_accuracy_guarantees: bool, + cost_inputs: CostInputs, +} + +/// A PromQL query `query` every `interval_ms` at phase `phase_ms`; a range +/// query with `(step_ms, span_ms)`; within `accuracy` (JSON, `null` for +/// none). +fn promql( + query: &str, + interval_ms: u64, + phase_ms: u64, + range: Option<(u64, u64)>, + accuracy: Value, +) -> Value { + let mut entry = json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": interval_ms, "evaluation_phase": phase_ms}}, + "requirements": {}, + "predictability": {"predictable": {"known_at": null}}, + }); + if !accuracy.is_null() { + entry["requirements"]["accuracy"] = accuracy; + } + if let Some((step_ms, span_ms)) = range { + entry["range_evaluation"] = json!({"step_ms": step_ms, "span_ms": span_ms}); + } + entry +} + +/// `entries` as a PromQL workload over the gauge `m` and the counter `c` +/// (1,000 series each, one sample per second, 2,000 rows/s), raw data kept +/// or not, accuracy enforced, default unit costs. +fn over_m_and_c(name: &'static str, entries: Vec, raw_data_retained: bool) -> Workload { + let planning = serde_json::from_value(json!({ + "query_workload": {"language": "promql", "repeating_queries": entries}, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 2000.0}}, + "sources": { + "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, + "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + }, + }, + })) + .unwrap(); + Workload { + name, + planning, + sql_tables: None, + raw_data_retained, + ignore_accuracy_guarantees: false, + cost_inputs: CostInputs::default(), + } +} + +/// SQL `queries` every `interval_ms` over `flows(ts, src, dst, bytes)` +/// (1,000 rows/s), each within `accuracy`, raw data kept or not. +fn over_flows( + name: &'static str, + queries: &[&str], + interval_ms: u64, + accuracy: Value, + raw_data_retained: bool, +) -> Workload { + over_flows_holding( + name, + queries, + interval_ms, + accuracy, + raw_data_retained, + None, + ) +} + +/// [`over_flows`] with the table holding `rows_held` rows (`None`: +/// undeclared, so a query-time scan of the whole table cannot be costed). +fn over_flows_holding( + name: &'static str, + queries: &[&str], + interval_ms: u64, + accuracy: Value, + raw_data_retained: bool, + rows_held: Option, +) -> Workload { + let flows = json!({"name": "flows", "time_index": 0, "columns": [ + {"name": "ts", "type": "timestamp"}, + {"name": "src", "type": "utf8"}, + {"name": "dst", "type": "utf8"}, + {"name": "bytes", "type": "float64"}, + ]}); + let planning = serde_json::from_value(json!({ + "query_workload": { + "language": {"sql": "datafusion_sql"}, + "repeating_queries": queries.iter().map(|query| { + let mut entry = json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": interval_ms, "evaluation_phase": 0}}, + "requirements": {}, + "predictability": {"predictable": {"known_at": null}}, + }); + if !accuracy.is_null() { + entry["requirements"]["accuracy"] = accuracy.clone(); + } + entry + }).collect::>(), + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"flows": match rows_held { + Some(rows_held) => json!({"series_count": 1000, "rows_held": rows_held}), + None => json!({"series_count": 1000}), + }}, + }, + })) + .unwrap(); + Workload { + name, + planning, + sql_tables: Some(vec![flows.to_string()]), + raw_data_retained, + ignore_accuracy_guarantees: false, + cost_inputs: CostInputs::default(), + } +} + +/// Unit costs that charge what the default ones leave at 0: 50 µs of +/// runtime overhead per query request and 1e-9 per byte-second on disk. +fn overhead_and_disk_costs() -> CostInputs { + let mut settings = CostInputs::default(); + settings.measured_operation_costs.ns_per_evaluation = 50_000.0; + settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost { + cost_per_disk_byte_second: 1e-9, + ..ResourceUnitCost::illustrative() + }); + settings +} + +/// What the stages produced for a workload. +struct Run { + physical: PhysicalASAPMemoCandidates, + entries: Vec, + data: DataWorkload, + selection: Result<(PhysicalASAPDAG, Selection), SelectionError>, +} + +/// Runs the stages over `w` through Stage 3 (`select_plan`). +fn run(w: &Workload) -> Run { + let entries: Vec<_> = w.planning.query_workload.entries().collect(); + let data = w.planning.data_workload.clone().unwrap(); + let roots = match &w.sql_tables { + Some(tables) => lower_sql_roots(&w.planning, &sql_catalog(tables).unwrap().0).unwrap(), + None => lower_promql_roots(&w.planning).unwrap(), + }; + let capabilities = DeploymentCapabilities { + raw_data_retained: w.raw_data_retained, + ..Default::default() + }; + let models = PlanningModels::builtin() + .with_cost_inputs(&w.cost_inputs) + .with_deployment_capabilities(&capabilities) + .with_ignore_accuracy_guarantees(w.ignore_accuracy_guarantees); + let stages = run_stages_until(roots, &entries, &data, models, RunUntil::Stage3) + .unwrap_or_else(|error| panic!("{}: planning failed before Stage 3: {error}", w.name)); + let selection = stages.run.selection; + let Some(physical) = stages.run.physical_asap_candidates else { + panic!("{}: no Stage 2 output: {:?}", w.name, selection.err()); + }; + Run { + physical, + entries, + data, + selection, + } +} + +/// The oracle over `run`; panics, so the test reports "cannot run", when +/// the memo is too large or has a shape the oracle does not model. +fn oracle(w: &Workload, run: &Run) -> Enumeration { + let enumeration = enumerate_plans( + &run.physical, + &run.entries, + &run.data, + &w.cost_inputs, + MAX_CHOICES, + ) + .unwrap_or_else(|error| panic!("{}: the oracle cannot run: {error}", w.name)); + println!( + "{}: {} choices, {} refused, {} not costable (first: {:?}), {} installable, {} not installable", + w.name, + enumeration.choices, + enumeration.refused, + enumeration.not_costable, + enumeration.first_cost_error, + enumeration.installable, + enumeration.not_installable + ); + enumeration +} + +/// `plan_cost` of `plan` per second under `w`'s cost inputs. +fn plan_cost(w: &Workload, run: &Run, plan: &PhysicalASAPDAG) -> f64 { + let (cost, _) = asap_cost::plan_cost(plan, &run.entries, &run.data, &w.cost_inputs) + .unwrap_or_else(|error| panic!("{}: plan_cost: {error:?}", w.name)); + cost.cost_per_second.expect("cost model 1") +} + +/// One line per node: id, candidate, operator, timing, lifetime, +/// retention, inputs. +fn describe(plan: &PhysicalASAPDAG) -> String { + plan.nodes + .iter() + .map(|node| { + let inputs: Vec = plan + .edges + .iter() + .filter(|edge| edge.consumer == node.id) + .map(|edge| edge.producer) + .collect(); + format!( + " {} v{:?} {} {:?} {:?} retention_ms={:?} <- {inputs:?}", + node.id, + node.selected_physical_candidate.map(|v| v.0), + node.payload.kind_name(), + node.output_state.timing, + node.lifetime, + node.retention_ms, + ) + }) + .collect::>() + .join("\n") +} + +/// The planner's own install rules over `plan`. +fn install_error(plan: &PhysicalASAPDAG) -> Option { + plan.materialized_views() + .err() + .or_else(|| plan.raw_data_retention().err()) + .map(|error| format!("{error:?}")) +} + +/// The optimality contract over `w`: Stage 3 returns a plan exactly when +/// some plan installs, the plan installs, and its `plan_cost` is the least +/// of any installable plan's. +fn assert_selects_the_cheapest_installable_plan(w: &Workload) { + let run = run(w); + let found = oracle(w, &run); + match (&found.cheapest_installable, &run.selection) { + (None, Err(_)) => {} + (None, Ok((plan, _))) => panic!( + "{}: no plan installs, yet Stage 3 selected one ({:?}):\n{}", + w.name, + install_error(plan), + describe(plan) + ), + (Some(best), Err(error)) => panic!( + "{}: Stage 3 selected no plan ({error}); the cheapest installable plan costs {}:\n{}", + w.name, + best.cost_per_second, + describe(&best.plan) + ), + (Some(best), Ok((plan, _))) => { + if let Some(error) = install_error(plan) { + panic!( + "{}: the selected plan does not install ({error}):\n{}", + w.name, + describe(plan) + ); + } + let selected = plan_cost(w, &run, plan); + let mut least_retention = plan.clone(); + set_least_retention(&mut least_retention, &run.entries); + let selected_at_least_retention = plan_cost(w, &run, &least_retention); + println!( + "{}: selected {selected}, cheapest installable {} ({} at the least retention)", + w.name, best.cost_per_second, selected_at_least_retention + ); + assert!( + selected <= best.cost_per_second * (1.0 + RELATIVE_TOLERANCE), + "{}: the selected plan costs {selected}/s, the cheapest installable plan {}/s \ + (+{:.3}%; the selected plan at the least retention its readers need: {}/s)\n\ + selected:\n{}\ncheapest:\n{}", + w.name, + best.cost_per_second, + 100.0 * (selected / best.cost_per_second - 1.0), + selected_at_least_retention, + describe(plan), + describe(&best.plan) + ); + } + } +} + +// --------------------------------------------------------------------------- +// The corpus: small workloads whose plans the oracle enumerates. + +fn corpus() -> Vec { + let none = Value::Null; + let mut workloads = vec![ + over_m_and_c( + "p01_sum_5m_raw", + vec![promql( + "sum_over_time(m[5m])", + 60_000, + 0, + None, + none.clone(), + )], + true, + ), + over_m_and_c( + "p02_sum_5m_noraw", + vec![promql( + "sum_over_time(m[5m])", + 60_000, + 0, + None, + none.clone(), + )], + false, + ), + over_m_and_c( + "p03_rate_5m_raw", + vec![promql("sum(rate(c[5m]))", 60_000, 0, None, none.clone())], + true, + ), + over_m_and_c( + "p04_max_14m_every_7m_noraw", + vec![promql( + "max_over_time(m[14m])", + 420_000, + 0, + None, + none.clone(), + )], + false, + ), + over_m_and_c( + "p05_count_1m_raw", + vec![promql( + "count_over_time(m[1m])", + 60_000, + 0, + None, + none.clone(), + )], + true, + ), + over_m_and_c( + "p06_two_phases_one_grid_raw", + vec![ + promql("sum_over_time(m[1m])", 20_000, 10_000, None, none.clone()), + promql("max_over_time(m[1m])", 10_000, 0, None, none.clone()), + ], + true, + ), + over_m_and_c( + "p07_range_query_1h_span_raw", + vec![promql( + "sum_over_time(m[5m])", + 60_000, + 0, + Some((60_000, 3_600_000)), + none.clone(), + )], + true, + ), + over_m_and_c( + "p08_two_filters_one_source_raw", + vec![ + promql( + r#"sum_over_time(m{job="a"}[5m])"#, + 60_000, + 0, + None, + none.clone(), + ), + promql( + r#"sum_over_time(m{job="b"}[5m])"#, + 60_000, + 0, + None, + none.clone(), + ), + ], + true, + ), + over_m_and_c( + "p09_quantile_rank_target_noraw", + vec![promql( + "quantile_over_time(0.9, m[5m])", + 60_000, + 0, + None, + json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}), + )], + false, + ), + over_m_and_c( + "p10_offset_1h_hourly_plus_plain_noraw", + vec![ + promql( + "sum_over_time(m[1m] offset 1h)", + 3_600_000, + 0, + None, + none.clone(), + ), + promql("sum_over_time(m[1m])", 10_000, 0, None, none.clone()), + ], + false, + ), + over_flows( + "s01_sql_windowed_sum", + &["SELECT sum(bytes) FROM flows WHERE ts > now() - INTERVAL '5' MINUTE"], + 10_000, + none.clone(), + true, + ), + ]; + let mut with_disk = over_m_and_c( + "p11_two_filters_overhead_and_disk_raw", + vec![ + promql( + r#"sum_over_time(m{job="a"}[5m])"#, + 60_000, + 0, + None, + none.clone(), + ), + promql( + r#"sum_over_time(m{job="b"}[5m])"#, + 60_000, + 0, + None, + none.clone(), + ), + ], + true, + ); + with_disk.cost_inputs = overhead_and_disk_costs(); + workloads.push(with_disk); + workloads.extend(regressions()); + workloads +} + +/// The regression workloads of the installable-selection and cost +/// findings. +fn regressions() -> Vec { + let none = Value::Null; + vec![ + // y13: an ingestion-time `TimeShift(90s)` view over the arriving + // rows was selected, under-costed, and refused at install + // (`ViewReadsUnwindowedSource`). + y13(), + // An ingestion-time `Scan` feeding a 60 s chain and a 420 s chain: + // charged once by Stage 3, once per window by `plan_cost`. + over_m_and_c( + "r5_scan_feeding_two_chain_widths_noraw", + vec![ + promql("sum_over_time(m[5m])", 60_000, 0, None, none.clone()), + promql("max_over_time(m[14m])", 420_000, 0, None, none.clone()), + ], + false, + ), + sql_table_without_window(), + sql_table_without_window_holding_rows(), + subquery_over_raw_range(), + negative_offset("b08_negative_offset_noraw", false), + negative_offset("b09_negative_offset_raw", true), + ] +} + +/// y13 (also the installable-selection design's example): +/// `sum_over_time(m[5m] offset 90s)` every 60 s, raw data not kept. +fn y13() -> Workload { + over_m_and_c( + "y13_offset_90s_noraw", + vec![promql( + "sum_over_time(m[5m] offset 90s)", + 60_000, + 0, + None, + Value::Null, + )], + false, + ) +} + +/// A SQL percentile over a table with no time window, every 10 s, rank +/// error 0.01: an ingestion-time stored `Project` over the whole table +/// was selected and refused at install (`ViewReadsUnwindowedSource`). The +/// table declares no `rows_held`, so no query-time plan of the whole table +/// can be costed either: no plan installs, and Stage 3 must return an +/// error. +fn sql_table_without_window() -> Workload { + over_flows( + "sql_percentile_whole_table", + &["SELECT approx_percentile_cont(bytes, 0.9) FROM flows"], + 10_000, + json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}), + true, + ) +} + +/// [`sql_table_without_window`] over a table declared to hold 1,000,000 +/// rows, so a query-time plan reading the whole table can be costed. +fn sql_table_without_window_holding_rows() -> Workload { + over_flows_holding( + "sql_percentile_whole_table_rows_held", + &["SELECT approx_percentile_cont(bytes, 0.9) FROM flows"], + 10_000, + json!({"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01}), + true, + Some(1_000_000), + ) +} + +/// The smallest shape of the o11y subquery workloads: a maximum over an +/// hour of 1 m steps of a rate over raw data, every 60 s, raw data kept. +/// A query-time `TimeRange` over the raw `Scan` under the subquery was +/// selected and refused at install (`UnsupportedRawDataWindow`); on this +/// head every plan of this memo installs. +fn subquery_over_raw_range() -> Workload { + over_m_and_c( + "subquery_max_of_rate_1h_raw", + vec![promql( + "max_over_time(sum(rate(c[5m]))[1h:1m])", + 60_000, + 0, + None, + Value::Null, + )], + true, + ) +} + +/// o11y-bench q05's shape over `c`: the maximum over 6 h of 1 m steps of +/// a ratio of two rates over raw data, every 60 s, raw data kept. +fn ratio_subquery_over_raw_ranges() -> Workload { + over_m_and_c( + "subquery_max_of_rate_ratio_6h_raw", + vec![promql( + r#"max_over_time((sum(rate(c{job="a"}[5m])) / sum(rate(c[5m])))[6h:1m])"#, + 60_000, + 0, + None, + Value::Null, + )], + true, + ) +} + +/// b08 (`raw` false) and b09 (`raw` true): `sum_over_time(m[5m] offset +/// -5m)` every 60 s reads a window ending after the evaluation. +fn negative_offset(name: &'static str, raw: bool) -> Workload { + over_m_and_c( + name, + vec![promql( + "sum_over_time(m[5m] offset -5m)", + 60_000, + 0, + None, + Value::Null, + )], + raw, + ) +} + +fn workload(name: &str) -> Workload { + corpus() + .into_iter() + .find(|w| w.name == name) + .unwrap_or_else(|| panic!("no workload {name}")) +} + +// --------------------------------------------------------------------------- +// Optimality: the selected plan is the cheapest installable plan. + +macro_rules! cheapest_installable_tests { + ($($test:ident => $name:literal,)*) => {$( + #[test] + fn $test() { + assert_selects_the_cheapest_installable_plan(&workload($name)); + } + )*}; +} + +cheapest_installable_tests! { + cheapest_p01_sum_5m_raw => "p01_sum_5m_raw", + cheapest_p02_sum_5m_noraw => "p02_sum_5m_noraw", + cheapest_p03_rate_5m_raw => "p03_rate_5m_raw", + cheapest_p04_max_14m_every_7m_noraw => "p04_max_14m_every_7m_noraw", + cheapest_p05_count_1m_raw => "p05_count_1m_raw", + cheapest_p06_two_phases_one_grid_raw => "p06_two_phases_one_grid_raw", + cheapest_p07_range_query_1h_span_raw => "p07_range_query_1h_span_raw", + cheapest_p08_two_filters_one_source_raw => "p08_two_filters_one_source_raw", + cheapest_p09_quantile_rank_target_noraw => "p09_quantile_rank_target_noraw", + cheapest_p10_offset_1h_hourly_plus_plain_noraw => "p10_offset_1h_hourly_plus_plain_noraw", + cheapest_p11_two_filters_overhead_and_disk_raw => "p11_two_filters_overhead_and_disk_raw", + cheapest_s01_sql_windowed_sum => "s01_sql_windowed_sum", + cheapest_y13_offset_90s_noraw => "y13_offset_90s_noraw", + cheapest_r5_scan_feeding_two_chain_widths_noraw => "r5_scan_feeding_two_chain_widths_noraw", + cheapest_sql_percentile_whole_table => "sql_percentile_whole_table", + cheapest_sql_percentile_whole_table_rows_held => "sql_percentile_whole_table_rows_held", + cheapest_subquery_max_of_rate_1h_raw => "subquery_max_of_rate_1h_raw", + cheapest_b08_negative_offset_noraw => "b08_negative_offset_noraw", + cheapest_b09_negative_offset_raw => "b09_negative_offset_raw", +} + +/// Reversing the order of a workload's queries changes no plan's cost, so +/// the cheapest installable plan's cost is the same: Stage 3's selected +/// cost is too. +#[test] +fn the_selected_cost_does_not_depend_on_query_order() { + for name in [ + "p06_two_phases_one_grid_raw", + "p08_two_filters_one_source_raw", + "p10_offset_1h_hourly_plus_plain_noraw", + "r5_scan_feeding_two_chain_widths_noraw", + ] { + let w = workload(name); + let mut reversed = w.clone(); + if let Some(queries) = reversed.planning.query_workload.repeating_queries.as_mut() { + queries.reverse(); + } + let cost = |w: &Workload| { + let run = run(w); + let (plan, _) = run + .selection + .as_ref() + .unwrap_or_else(|e| panic!("{name}: {e}")); + plan_cost(w, &run, plan) + }; + let (forward, backward) = (cost(&w), cost(&reversed)); + assert!( + (forward - backward).abs() <= RELATIVE_TOLERANCE * forward.abs().max(1e-12), + "{name}: {forward}/s in workload order, {backward}/s reversed" + ); + } +} + +// --------------------------------------------------------------------------- +// The oracle's own checks: its space holds Stage 3's plan. + +/// For every corpus workload Stage 3 plans and the oracle can enumerate, +/// the oracle built Stage 3's plan (the same executions over the same +/// edges, `plan_signature`): otherwise the oracle misses plans Stage 3 may +/// select, and the optimality contract proves nothing. +#[test] +fn the_oracle_space_holds_every_selected_plan() { + let mut failures = Vec::new(); + for w in corpus() { + let run = run(&w); + let Ok((plan, _)) = &run.selection else { + continue; + }; + // A memo the oracle cannot enumerate is reported, not checked: its + // own `cheapest_*` test fails as "cannot run". + let found = match enumerate_plans( + &run.physical, + &run.entries, + &run.data, + &w.cost_inputs, + MAX_CHOICES, + ) { + Ok(found) => found, + Err(error) => { + println!("{}: not checked, the oracle cannot run: {error}", w.name); + continue; + } + }; + if !found.plan_signatures.contains(&plan_signature(plan)) { + failures.push(format!( + "{}: the oracle did not build the selected plan {}:\n{}", + w.name, + plan_signature(plan), + describe(plan) + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n\n")); +} + +// --------------------------------------------------------------------------- +// Feasibility and constraints of every selected plan. + +/// Every corpus workload Stage 3 plans, with its plan. +fn selected_plans() -> Vec<(Workload, Run)> { + corpus() + .into_iter() + .map(|w| { + let run = run(&w); + (w, run) + }) + .filter(|(_, run)| run.selection.is_ok()) + .collect() +} + +/// Every selected plan passes the planner's install rules. +#[test] +fn every_selected_plan_passes_the_install_rules() { + let failures: Vec = selected_plans() + .iter() + .filter_map(|(w, run)| { + let (plan, _) = run.selection.as_ref().unwrap(); + install_error(plan).map(|error| format!("{}: {error}\n{}", w.name, describe(plan))) + }) + .collect(); + assert!(failures.is_empty(), "{}", failures.join("\n\n")); +} + +/// A query-time result that is not materialized streams to one reader: no +/// selected node with ℓ = None at query time is read by two edges or by an +/// edge and a query. +#[test] +fn no_unmaterialized_query_time_node_has_two_readers() { + let mut failures = Vec::new(); + for (w, run) in selected_plans() { + let (plan, _) = run.selection.as_ref().unwrap(); + for node in &plan.nodes { + if node.output_state.timing != ExecutionTiming::QueryTime + || node.lifetime != Lifetime::None + { + continue; + } + let readers = plan + .edges + .iter() + .filter(|edge| edge.producer == node.id) + .count() + + plan.roots.iter().filter(|&&root| root == node.id).count() + + plan + .scalar_roots + .iter() + .flat_map(|(_, expression)| expression.operator_refs()) + .filter(|&&id| id == node.id) + .count(); + if readers > 1 { + failures.push(format!( + "{}: node {} has {readers} readers\n{}", + w.name, + node.id, + describe(plan) + )); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n\n")); +} + +/// Each plan edge is an admissible physical edge into that input of its +/// consumer's candidate. +#[test] +fn every_selected_edge_is_admissible() { + let mut failures = Vec::new(); + for (w, run) in selected_plans() { + let (plan, _) = run.selection.as_ref().unwrap(); + let candidate = |id: PhysicalASAPNodeId| { + plan.nodes + .iter() + .find(|node| node.id == id) + .unwrap() + .selected_physical_candidate + .unwrap() + }; + for node in &plan.nodes { + let consumer = candidate(node.id); + let producers: Vec = node + .payload + .children() + .iter() + .map(|&&producer| candidate(producer)) + .collect(); + for (input, producer) in producers.into_iter().enumerate() { + let admissible = run + .physical + .admissible_edges_into(consumer, input) + .iter() + .any(|&edge| run.physical.edge(edge).producer_candidate_id == producer); + if !admissible { + failures.push(format!( + "{}: node {} input {input} reads v{} over no admissible edge", + w.name, node.id, producer.0 + )); + } + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// (X): no query reads, anywhere below its root, a candidate it must read +/// only exactly. +#[test] +fn no_query_reads_a_sketch_of_a_group_it_must_read_exactly() { + let mut failures = Vec::new(); + for (w, run) in selected_plans() { + let (plan, _) = run.selection.as_ref().unwrap(); + for (id, queries) in queries_reaching(plan) { + let node = plan.nodes.iter().find(|node| node.id == id).unwrap(); + let v = node.selected_physical_candidate.unwrap(); + let logical = run.physical.physical_candidates[v.0].logical_candidate_id; + for query in queries { + if run + .physical + .logical_candidates + .query_must_not_read(query, logical) + { + failures.push(format!( + "{}: query {query} reads node {id} (v{})", + w.name, v.0 + )); + } + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// Stage 3 retains at least what the plan's readers need (the least +/// retention the oracle sets): otherwise the installer refuses the plan or +/// a later evaluation misses rows. +#[test] +fn stage3_retains_at_least_what_its_plans_readers_need() { + let mut failures = Vec::new(); + for (w, run) in selected_plans() { + let (plan, _) = run.selection.as_ref().unwrap(); + let mut least = plan.clone(); + set_least_retention(&mut least, &run.entries); + for (selected, needed) in plan.nodes.iter().zip(&least.nodes) { + let short = match (selected.retention_ms, needed.retention_ms) { + (Some(selected), Some(needed)) => selected < needed, + (Some(_), None) => true, + (None, _) => false, + }; + if short { + failures.push(format!( + "{}: node {} retains {:?} ms, its readers need {:?}", + w.name, selected.id, selected.retention_ms, needed.retention_ms + )); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// Stage 3's objective is the `plan_cost` of the plan it selects, over the +/// whole corpus (the example tests in `stage3_objective` check it on +/// chosen shapes). +#[test] +fn the_objective_is_the_plan_cost_over_the_corpus() { + let mut failures = Vec::new(); + for (w, run) in selected_plans() { + let (plan, selection) = run.selection.as_ref().unwrap(); + let cost = plan_cost(&w, &run, plan); + let objective = selection.selected_plan_cost_per_second; + if (objective - cost).abs() > 1e-9 * cost.abs().max(1e-12) { + failures.push(format!( + "{}: objective {objective} != plan_cost {cost}", + w.name + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +// --------------------------------------------------------------------------- +// Installable selection on the regression workloads. + +/// The candidates Stage 3 reports as `NotInstallable`. +fn not_installable(selection: &Selection) -> Vec { + selection + .pruned_candidates + .iter() + .filter(|pruned| pruned.reason == PruneReason::NotInstallable) + .map(|pruned| pruned.candidate) + .collect() +} + +/// When the cheapest plan of all does not install, the MILP's first +/// optimum is refused: Stage 3 reports the candidate of the node the +/// refused rule names as `NotInstallable`, and selects a plan that +/// installs at the cheapest installable cost. +fn assert_refused_candidate_is_reported(w: &Workload) { + let run = run(w); + let found = oracle(w, &run); + let cheapest = found.cheapest.as_ref().expect("a costed plan"); + let error = cheapest + .install_error + .as_ref() + .unwrap_or_else(|| panic!("{}: the cheapest plan installs; nothing to refuse", w.name)); + let node = error.node_id().unwrap_or_else(|| { + panic!( + "{}: the cheapest plan's error {error:?} is not subtree-local", + w.name + ) + }); + let refused = cheapest.plan.nodes.iter().find(|n| n.id == node).unwrap(); + let refused = run.physical.physical_candidates[refused.selected_physical_candidate.unwrap().0]; + let (plan, selection) = run + .selection + .as_ref() + .unwrap_or_else(|e| panic!("{}: Stage 3 selected no plan: {e}", w.name)); + assert!( + not_installable(selection).contains(&refused), + "{}: {refused:?} (node {node} of the cheapest plan, {error:?}) is not reported NotInstallable: {:?}", + w.name, + selection.pruned_candidates + ); + assert_eq!(install_error(plan), None, "{}:\n{}", w.name, describe(plan)); +} + +/// The SQL whole-table case: the cheapest plan stores a `Project` over +/// the whole table at ingestion time, which no ingestion run can refresh. +#[test] +fn a_view_over_a_whole_table_is_reported_not_installable() { + assert_refused_candidate_is_reported(&sql_table_without_window_holding_rows()); +} + +/// o11y-bench q05's shape (too many plans for the oracle): the plan +/// Stage 3 returns passes the install rules (I1 without the oracle). +#[test] +fn a_ratio_subquery_over_raw_ranges_selects_an_installable_plan() { + let w = ratio_subquery_over_raw_ranges(); + let run = run(&w); + match &run.selection { + Ok((plan, _)) => assert_eq!(install_error(plan), None, "{}:\n{}", w.name, describe(plan)), + Err(error) => println!("{}: no plan ({error})", w.name), + } +} + +/// No candidate Stage 3 reports `NotInstallable` is in the plan it +/// returns, on any corpus workload. +#[test] +fn no_candidate_reported_not_installable_is_selected() { + let mut failures = Vec::new(); + for (w, run) in selected_plans() { + let (plan, selection) = run.selection.as_ref().unwrap(); + let refused = not_installable(selection); + for node in &plan.nodes { + let v = node.selected_physical_candidate.unwrap(); + if refused.contains(&run.physical.physical_candidates[v.0]) { + failures.push(format!( + "{}: node {} (v{}) is reported NotInstallable", + w.name, node.id, v.0 + )); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// y13 without raw data: Stage 3's plan installs and is the cheapest +/// installable plan. The ingestion-time `TimeShift(90s)` view over the +/// arriving rows is not installable whatever it costs. +#[test] +fn y13_selects_an_installable_plan_and_no_shift_view_over_arriving_rows() { + let w = y13(); + let run = run(&w); + let (plan, _) = run.selection.as_ref().expect("a plan"); + assert_eq!(install_error(plan), None, "{}", describe(plan)); + let shift_views: Vec = plan + .nodes + .iter() + .filter(|node| { + node.output_state.timing == ExecutionTiming::IngestionTime + && matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) + && node.payload.kind_name() == "TimeShift" + }) + .map(|node| node.id) + .collect(); + assert!( + shift_views.is_empty(), + "shift views {shift_views:?}:\n{}", + describe(plan) + ); +} + +/// b08 and b09: no plan reads a window ending after the evaluation, so +/// none installs (the oracle finds none) and Stage 3 returns an error. +#[test] +fn a_negative_offset_has_no_installable_plan_and_stage3_returns_an_error() { + for w in [negative_offset("b08", false), negative_offset("b09", true)] { + let run = run(&w); + let found = oracle(&w, &run); + assert!( + found.cheapest_installable.is_none(), + "{}: the oracle installs\n{}", + w.name, + describe(&found.cheapest_installable.as_ref().unwrap().plan) + ); + assert!( + run.selection.is_err(), + "{}: Stage 3 selected a plan", + w.name + ); + } +} diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index e3a8182ae..242d53898 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -321,5 +321,21 @@ pub enum InstallableDagError { ScanRetentionTooShort { node_id: PhysicalASAPNodeId }, } +impl InstallableDagError { + /// The node whose subtree the refused rule is about, when the rule + /// depends only on that subtree (`UnsupportedRawDataWindow`'s + /// `TimeRange`, `ViewReadsUnwindowedSource`, + /// `IngestionBuildReadsWindowsOfDifferentOffsets`, + /// `ViewWithoutCoverage`): Stage 3 selects again without that node's + /// candidate. `None` for a rule that depends on the node's readers (a + /// retention, which Stage 3 sets from them) or on the whole DAG. + /// + /// Example: `ViewReadsUnwindowedSource { node_id: 2 }` is `Some(2)`; + /// `ViewRetentionTooShort { node_id: 2 }` is `None`. + pub fn node_id(&self) -> Option { + todo!("installable selection: implement after the tests are reviewed") + } +} + mod raw_data; mod views; diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index d5b4227fb..37f9c3b18 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -141,4 +141,13 @@ pub enum PruneReason { /// Stage 3: `asap-cost` could not cost it, so the selection MILP leaves /// it out. UsageNotCostable, + /// Stage 3: a selected plan with it was refused by a planner-owned + /// install rule over its subtree (`PhysicalASAPDAG::materialized_views`); + /// the MILP was solved again without it. + /// + /// Example: `SELECT approx_percentile_cont(bytes, 0.9) FROM flows` every + /// 10 s, `flows` declaring its `rows_held`: the ingestion-time stored + /// `Project` over the whole table fails `ViewReadsUnwindowedSource`; the + /// re-solve picks a query-time plan. + NotInstallable, } diff --git a/crates/types/tests/installable_dag_error.rs b/crates/types/tests/installable_dag_error.rs new file mode 100644 index 000000000..c42df2d1c --- /dev/null +++ b/crates/types/tests/installable_dag_error.rs @@ -0,0 +1,63 @@ +//! Which install rules Stage 3 can act on by selecting again: a rule over +//! one node's subtree names that node (`InstallableDagError::node_id`); a +//! rule over a node's readers or over the whole DAG names none. +use asap_types::plan::materialized_view::InstallableDagError; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAGValidationError; + +/// Each variant with the node `node_id` returns: the subtree-local rules +/// name their node (for `UnsupportedRawDataWindow`, its `TimeRange`); the +/// retention rules, which depend on the readers Stage 3 sets the retention +/// from, and an invalid DAG name none. +#[test] +fn node_id_names_the_node_of_a_subtree_local_rule_only() { + let cases: Vec<(InstallableDagError, Option)> = vec![ + ( + InstallableDagError::UnsupportedRawDataWindow { + time_range_node_id: 3, + reason: "a subquery or a TimeShift above the range", + }, + Some(3), + ), + ( + InstallableDagError::ViewReadsUnwindowedSource { node_id: 2 }, + Some(2), + ), + ( + InstallableDagError::ViewWithoutCoverage { + node_id: 4, + reason: "not a summary state".into(), + }, + Some(4), + ), + ( + InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { + node_id: 5, + end_offsets_ms: vec![0, 90_000], + }, + Some(5), + ), + ( + InstallableDagError::ViewWithoutRetention { node_id: 1 }, + None, + ), + ( + InstallableDagError::ViewRetentionTooShort { node_id: 1 }, + None, + ), + ( + InstallableDagError::ScanWithoutRetention { node_id: 0 }, + None, + ), + ( + InstallableDagError::ScanRetentionTooShort { node_id: 0 }, + None, + ), + ( + InstallableDagError::InvalidDag(PhysicalASAPDAGValidationError::Cycle), + None, + ), + ]; + for (error, expected) in cases { + assert_eq!(error.node_id(), expected, "{error:?}"); + } +} diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index a5afa977b..70e850891 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -3,6 +3,7 @@ //! One file: `cargo test -p asap-types --test types ::`. mod column_lineage; +mod installable_dag_error; mod memo_aggregate_coverage; mod memo_group_properties; mod physical_asap_dag;