diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index 0ddbd7b7a..304dd7814 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -336,5 +336,18 @@ pub struct ErrorContract { pub linear_bound_holds_up_to_per_input: Vec>, } +impl ErrorContract { + /// Whether any of input `input`'s error reaches the output + /// ($\beta_{c,i} > 0$). Where it does not, the input may be off in the + /// contract's metric whatever the output allows. + /// + /// Example: `count` outputs how many values it reads, whatever their + /// error ($\beta = 0$): false; `sum` passes its input's relative error + /// on ($\beta = 1$): true. + pub(crate) fn input_error_reaches_output(&self, input: usize) -> bool { + self.error_factor_per_input[input] > 0.0 + } +} + #[cfg(test)] mod tests; diff --git a/crates/accuracy/src/logical_edge/mod.rs b/crates/accuracy/src/logical_edge/mod.rs index c99686394..56542b462 100644 --- a/crates/accuracy/src/logical_edge/mod.rs +++ b/crates/accuracy/src/logical_edge/mod.rs @@ -73,7 +73,7 @@ impl ErrorMetricsOfMemo { let Some(producer_metric) = self.readout_error_metric(producer) else { return Ok(()); }; - return match self.error_metrics_allowed_on_inputs(consumer) { + return match self.error_metrics_allowed_on_input_edge(consumer, input) { Some(allowed) => metric_rule(producer_metric, &allowed), None => Ok(()), }; @@ -100,7 +100,7 @@ impl ErrorMetricsOfMemo { let failure_probability = required.map_or(1.0, |required| required.failure_probability); let measured = evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); - let allowed = self.error_metrics_allowed_on_inputs(consumer); + let allowed = self.error_metrics_allowed_on_input_edge(consumer, input); // The theoretical guarantee before its failure-probability check, // computed once for both checks below. let theoretical = (measured.is_none() && (allowed.is_some() || required.is_some())) diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 51dd3e7e9..9d15c30be 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -46,7 +46,12 @@ impl ErrorMetricsOfMemo { /// input those its output allows that its error contract passes to it; /// a group the union over its consumers. These are the metrics of the /// readout caps' walk (`readout_caps`), which also reaches a group - /// through inputs the output does not depend on. + /// through inputs the output does not depend on. A metric only such an + /// input would allow is not added here: a readout's edge into such a + /// consumer is admitted by [`Self::error_metrics_allowed_on_input`], + /// and widening a whole group for it would admit every alternative + /// readout of the group (`avg` of a relative quantile: 62 → 455 + /// MILP variables, the same plan). pub fn of(memo: &Memo, order: &[MemoGroupId]) -> Self { let error_contract: Vec> = memo .groups() @@ -111,46 +116,63 @@ impl ErrorMetricsOfMemo { self.readout_error_metric[candidate.group.0 as usize][candidate.index_in_group] } - /// The metrics among `metrics` that `consumer`'s error contract passes - /// to its inputs: those an input of it may be off in when its output - /// may be off in `metrics`. - fn error_metrics_passed_to_inputs( + /// The metrics `consumer`'s input `input` may be off in when its + /// output may be off in `output_allowed`, the metric rule: those of + /// `output_allowed` its error contract passes to its inputs, and every + /// metric in which the input's error does not reach the output + /// ([`ErrorContract::input_error_reaches_output`]). In + /// `AccuracyMetric::ALL` order. + /// + /// Example: a readout's contract passes every metric, so a readout + /// allows its output's; `count` (error factor 0) allows every metric, + /// even under an exact target; a sum of values of unknown sign under a + /// relative target allows none. + pub(crate) fn error_metrics_allowed_on_input( &self, consumer: CandidateForMemoGroupId, - metrics: &[AccuracyMetric], + input: usize, + output_allowed: &[AccuracyMetric], ) -> Vec { - metrics - .iter() - .copied() - .filter(|&metric| self.error_contract(consumer, metric).is_some()) + AccuracyMetric::ALL + .into_iter() + .filter(|&metric| { + self.error_contract(consumer, metric) + .is_some_and(|contract| { + output_allowed.contains(&metric) + || !contract.input_error_reaches_output(input) + }) + }) .collect() } - /// The metrics `consumer` allows an input's error in, the metric rule - /// on its input edges: those its group's output allows that its error - /// contract passes to its inputs (a readout's contract passes every - /// metric, so a readout allows its output's). `None` where no query - /// with a target reaches the consumer. - pub(crate) fn error_metrics_allowed_on_inputs( + /// [`Self::error_metrics_allowed_on_input`] at what `consumer`'s group's + /// output allows, the metric rule on its input edges; `None` where no + /// query with a target reaches the consumer. + pub(crate) fn error_metrics_allowed_on_input_edge( &self, consumer: CandidateForMemoGroupId, + input: usize, ) -> Option> { let allowed = self.error_metrics_allowed_at_output[consumer.group.0 as usize].as_ref()?; - Some(self.error_metrics_passed_to_inputs(consumer, allowed)) + Some(self.error_metrics_allowed_on_input(consumer, input, allowed)) } /// Per query: the groups below its root it may read only exactly, the /// metric rule per query where the edges apply it per group. Walking /// down the live candidates (not in `dead`) from each root with a /// target, which allows its target's metrics - /// ([`query_roots_with_target`]; an exact target none), a candidate passes to its inputs the - /// metrics its error contract passes; a group keeps, per query, the + /// ([`query_roots_with_target`]; an exact target none), a candidate + /// passes to each input [`Self::error_metrics_allowed_on_input`] (a + /// `count`'s input is never read exactly for its sake); a group keeps, + /// per query, the /// metrics every path to it passes. A group is exact-only for a query /// when no metric reaches it (one path through a consumer without a /// contract, a comparison or a sum, suffices) or when it holds a sketch /// readout whose error metric is not among them (`quantile_over_time` /// at a relative target reads no KLL readout, whose error is a rank - /// one); and so is every group below one. + /// one); and so is every group below one, except through an input + /// that may be off in some metric under an output allowing none (a + /// `count`'s). /// `exact_only_readout_groups_of_query` adds, per query, the readout /// groups the readout requirements list (a readout whose level another /// query gave, for a query whose failure probability has no per-value @@ -212,32 +234,36 @@ impl ErrorMetricsOfMemo { } } } - let passed = self.error_metrics_passed_to_inputs(consumer, &AccuracyMetric::ALL); - for &child in memo.candidate(consumer).expression.operator.children() { + for (input, &child) in memo + .candidate(consumer) + .expression + .operator + .children() + .into_iter() + .enumerate() + { for (&query, metrics) in &allowed_here { - let passed_here = metrics - .iter() - .copied() - .filter(|metric| passed.contains(metric)) - .collect(); - keep_common(&mut allowed, child, query, passed_here); + let passed = self.error_metrics_allowed_on_input(consumer, input, metrics); + keep_common(&mut allowed, child, query, passed); } } } } - // Every group below an exact-only group is read exactly too. + // Every group below an exact-only group is read exactly too, except + // through an input that may be off in some metric under an output + // allowing none (a `count`'s). for &group in order.iter().rev() { for groups in &mut exact_only { if groups.contains(&group) { for consumer in live_candidates(group) { - groups.extend( - memo.candidate(consumer) - .expression - .operator - .children() - .into_iter() - .copied(), - ); + let children = memo.candidate(consumer).expression.operator.children(); + groups.extend(children.into_iter().enumerate().filter_map( + |(input, &child)| { + self.error_metrics_allowed_on_input(consumer, input, &[]) + .is_empty() + .then_some(child) + }, + )); } } } diff --git a/crates/accuracy/src/metric_rule/tests.rs b/crates/accuracy/src/metric_rule/tests.rs index b76d523bb..a4c186fec 100644 --- a/crates/accuracy/src/metric_rule/tests.rs +++ b/crates/accuracy/src/metric_rule/tests.rs @@ -25,12 +25,13 @@ fn allowed_at_output(memo: &Memo, group: MemoGroupId) -> Option Option> { - error_metrics(memo).error_metrics_allowed_on_inputs(consumer) + error_metrics(memo).error_metrics_allowed_on_input_edge(consumer, input) } /// $\alpha^{\log}$ of the edge from `producer` into input `input` of @@ -93,7 +94,7 @@ fn a_ratio_reads_no_rank_error() { query(&mut memo, ratio, Some(target)); let relative = Some(vec![AccuracyMetric::Relative]); assert_eq!(allowed_at_output(&memo, p99), relative); - assert_eq!(allowed_on_inputs(&memo, own_candidate(ratio)), relative); + assert_eq!(allowed_on_input(&memo, own_candidate(ratio), 0), relative); assert!(matches!( edge(&memo, own_candidate(p99), own_candidate(ratio), 0), Err(RootPruneReason::MetricRule { @@ -143,7 +144,7 @@ fn the_metric_rule_cuts_the_readout_edges_and_keeps_the_exact_division() { let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::ABSOLUTE); query(&mut memo, avg, Some(target)); let division = own_candidate(ratio); - assert_eq!(allowed_on_inputs(&memo, division), Some(vec![])); + assert_eq!(allowed_on_input(&memo, division, 0), Some(vec![])); let result = edge(&memo, division, own_candidate(sum), 0); assert!(result.is_ok(), "{result:?}"); for input in 0..2 { @@ -195,7 +196,7 @@ fn a_key_only_filter_keeps_the_ddsketch_readout_admissible() { let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); query(&mut memo, filtered, Some(target)); assert_eq!( - allowed_on_inputs(&memo, own_candidate(filtered)), + allowed_on_input(&memo, own_candidate(filtered), 0), Some(vec![AccuracyMetric::Relative]) ); let result = readout_edge(&memo, &p99.by_algorithm[0], own_candidate(filtered), 0); @@ -332,3 +333,75 @@ fn a_scalar_roots_operands_are_roots() { check(scalar_of, vec![AccuracyMetric::Relative], true); check(doubled, vec![], false); } + +/// `count(quantile_over_time(0.99, m[5m]))` at an exact target: the +/// count of the quantiles does not depend on their values (its error +/// factor on its input is 0), so the quantile, its readouts and their +/// states are not exact-only for the query: the count's own group is, +/// and the raw window and scan below are (read only by the exact +/// quantile and the sketch builds, which pass no error). A +/// `count` beside a `sum` in `avg = sum / count` is such a path: it no +/// longer makes the quantile exact-only for a relative `avg`. +#[test] +fn a_count_reads_its_input_in_any_metric() { + let mut memo = Memo::new(&DataWorkload::default()); + let p99 = quantile_readouts(&mut memo, 0.99); + let count = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99.result, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Count], + output_names: vec!["count".into()], + filters: vec![None], + having: None, + }), + ); + query(&mut memo, count, Some(AccuracyTarget::exact())); + let order = memo.topological_order().expect("acyclic"); + let exact_only = error_metrics(&memo).exact_only_groups_of_query( + &memo, + &order, + &BTreeSet::new(), + &BTreeMap::new(), + ); + let [exact_only] = exact_only.as_slice() else { + panic!("one query: {exact_only:?}"); + }; + assert!(exact_only.contains(&count)); + let mut quantile_groups = vec![p99.result]; + for readout in &p99.by_algorithm { + quantile_groups.extend([readout.state, readout.readout.group]); + } + for group in quantile_groups { + assert!(!exact_only.contains(&group), "{group:?} in {exact_only:?}"); + } + assert_eq!(exact_only.len(), 3, "{exact_only:?}"); +} + +/// A `count`'s output does not depend on its input's error (error factor +/// 0 in every metric), so the metric rule lets its input be off in any +/// metric, on its edges as in `exact_only_groups_of_query`. +/// `count(quantile_over_time(0.99, m[5m]))` with a relative target, the +/// count reading the KLL readout directly: the readout's rank error is not +/// one the output allows, but its edge into the `count` holds. +#[test] +fn a_counts_input_edge_from_a_readout_holds_in_a_metric_its_output_does_not_allow() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, _, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let count = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Count], + output_names: vec!["count".into()], + filters: vec![None], + having: None, + }), + ); + let relative = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, count, Some(relative)); + let result = edge(&memo, own_candidate(p99), own_candidate(count), 0); + assert!(result.is_ok(), "{result:?}"); +} diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index c0784392f..be3925b49 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -265,10 +265,10 @@ pub(crate) fn caps( for (input, &child) in children.iter().enumerate() { let child_caps = caps[child.0 as usize].get_or_insert_with(HashMap::new); child_caps.entry(metric).or_default(); - let factor = contract.error_factor_per_input[input]; // An input the output does not depend on (a count's, // β = 0) gets no cap from it. - if factor > 0.0 { + if contract.input_error_reaches_output(input) { + let factor = contract.error_factor_per_input[input]; let holds_up_to = contract.linear_bound_holds_up_to_per_input[input] .unwrap_or(f64::INFINITY); for (&query, &cap) in caps_per_query { diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 86ef8062c..4ea46e909 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -20,7 +20,7 @@ //! Stage 3 (`asap-stage3-plan-selection`) costs each candidate as one node //! with the same function and the rules `plan_cost` applies: //! [`plan::execution_demand::executions_per_second`], -//! [`maintained_window_ms`], [`read_back_ms`], +//! `RefreshWindow::computed_from` (an ingestion-time node's window), [`read_back_ms`], //! [`raw_data_disk_cost_per_second`] and //! [`request_overhead_cost_per_second`]. mod measured; @@ -40,8 +40,8 @@ pub use model::{ }; pub use node::operator_statistics::EdgeStatistics; pub use node::usage::{ - maintained_window_ms, node_resource_usage_and_cost, summary_type_name, NodeCost, - NodeResourceUsageAndCost, ProducerOutput, + node_resource_usage_and_cost, output_interval_ms, reads_one_batch_of, summary_type_name, + NodeCost, NodeResourceUsageAndCost, ProducerOutput, }; pub use plan::cost::{ plan_cost, raw_data_disk_cost_per_second, request_overhead_cost_per_second, PlanCost, diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 668681e7c..f347c1259 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -22,7 +22,7 @@ use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; use asap_types::plan::physical_asap_dag::{ - PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, + PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; use asap_types::stages::{Lifetime, StorageMedium}; use asap_types::summary_type::{SketchParams, SketchStatistic}; @@ -30,7 +30,7 @@ use asap_types::workload::bounds::{ declared_non_negative, declared_series, group_count_bound, source_name, source_rows_per_ms, unchanged_source_columns, }; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, ONE_SECOND_MS}; /// One node's cost per second of wall time, with how it was measured. A /// node is one operator node of the candidate's physical plan DAG @@ -123,8 +123,15 @@ pub struct ProducerOutput<'a> { /// column) values unchanged ([`unchanged_source_columns`]). pub unchanged_columns: BTreeSet<&'a str>, /// The tumbling windows its rows span: those a `TimeRange` selects - /// from a chain, 1 for any other node. + /// from a chain (through the operators that keep each row's window), + /// 1 for any other node. pub tumbling_windows: u64, + /// The tumbling-window width of the chain whose states its rows are + /// read from: a chain's own, kept through the operators over it that + /// keep each row's window (`Operator::keeps_input_window`: `TimeShift`, + /// `Filter`, `Project`), so a `TimeRange` over them selects windows; + /// `None` for any other node. + pub chain_tumbling_window_width_ms: Option, } /// What one node uses and outputs ([`node_resource_usage_and_cost`]): its @@ -146,8 +153,15 @@ pub struct NodeResourceUsageAndCost<'a> { /// column) values unchanged ([`unchanged_source_columns`]). pub unchanged_columns: BTreeSet<&'a str>, /// The tumbling windows its rows span: those a `TimeRange` selects - /// from a chain, 1 for any other node. + /// from a chain (through the operators that keep each row's window), + /// 1 for any other node. pub tumbling_windows: u64, + /// The tumbling-window width of the chain whose states its rows are + /// read from: a chain's own, kept through the operators over it that + /// keep each row's window (`Operator::keeps_input_window`: `TimeShift`, + /// `Filter`, `Project`), so a `TimeRange` over them selects windows; + /// `None` for any other node. + pub chain_tumbling_window_width_ms: Option, /// Work of one execution, in ms on one core. pub work_ms_per_execution: f64, /// CPU operations per second. @@ -187,71 +201,54 @@ impl<'a> NodeResourceUsageAndCost<'a> { family: self.family, unchanged_columns: self.unchanged_columns.clone(), tumbling_windows: self.tumbling_windows, + chain_tumbling_window_width_ms: self.chain_tumbling_window_width_ms, } } } -/// One second, in ms: an ingestion-time node runs over the rows arriving -/// in one second, and an ingestion-time view that maintains no window is -/// refreshed once per second. -const ONE_SECOND_MS: u64 = 1_000; +/// Whether `consumer` reads only one batch of `producer`'s output: it runs +/// at ingestion time and `producer` keeps raw rows stored at ingestion +/// (`PhysicalASAPDAGNode::keeps_raw_rows_at_ingestion`, over +/// `producer_input`, its first producer's operator), whose output is the +/// rows it retains, what a query-time reader reads. Such a reader reads +/// the arriving batch, as from a `Scan` that is not materialized; Stage 3 +/// and [`plan_cost`](crate::plan_cost) both cost it so. +pub fn reads_one_batch_of( + consumer: &PhysicalASAPDAGNode, + producer: &PhysicalASAPDAGNode, + producer_input: Option<&Operator>, +) -> bool { + consumer.output_state.timing == ExecutionTiming::IngestionTime + && producer.keeps_raw_rows_at_ingestion(producer_input) +} -/// The window an ingestion-time grouping `node` maintains, in ms: a -/// tumbling-window chain's tumbling window, a `TimeRange`'s range, or the -/// first `TimeRange` or chain below it through filters, projections and -/// time shifts (a view over an ingestion-time chain is refreshed once per -/// chain window; `input_of` gives a node's first input, `operator_of` its -/// operator). `None` when there is none. Over plan nodes ([`plan_cost`]) -/// or memo groups (Stage 3, before any plan exists). -/// -/// Example: `Aggregate(Sum) <- Filter <- TimeRange(1m) <- Scan`: 60,000; -/// `TimeShift(1h) <- SummaryTumblingWindowChain(1m)`: 60,000. -/// -/// [`plan_cost`]: crate::plan_cost -pub fn maintained_window_ms<'o, N: Copy + Ord, C: 'o>( - node: N, - operator_of: impl Fn(N) -> &'o Operator, - input_of: impl Fn(N) -> Option, -) -> Option { - let range_ms = |range: &std::time::Duration| u64::try_from(range.as_millis()).ok(); - match operator_of(node) { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => return Some(*tumbling_window_width_ms), - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), - _ => {} - } - let mut seen = BTreeSet::new(); - let mut below = input_of(node)?; - while seen.insert(below) { - match operator_of(below) { - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => return Some(*tumbling_window_width_ms), - Operator::NonASAP( - NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, - ) => below = input_of(below)?, - _ => return None, - } - } - None +/// The time between two outputs of a node running `payload` that keeps +/// rows, which its retention is divided by: a chain's tumbling window, +/// else `window_ms` (the window it maintains), else one second of +/// arriving rows; at least 1 ms. Stage 3's $\pi_v$ is the cost of one +/// more of it. +pub fn output_interval_ms(payload: &Operator, window_ms: Option) -> u64 { + payload + .chain_tumbling_window_width_ms() + .unwrap_or_else(|| window_ms.unwrap_or(ONE_SECOND_MS)) + .max(1) } /// Measure and cost one node of a physical DAG (one step of /// [`plan_cost`]; Stage 3 calls it per candidate): `producers` are its -/// producers in edge order, `rate` its `(multiplier, per_second)` -/// ([`computations`], [`executions_per_second`]), and `window_ms` the time -/// window one execution covers: the window an ingestion-time node -/// maintains ([`maintained_window_ms`]), or a query-time `Scan`'s read, the +/// producers in its payload's input order (`Operator::children`), `rate` +/// its `(multiplier, per_second)` ([`computations`], +/// [`executions_per_second`]), and `window_ms` the time +/// window one execution covers: an ingestion-time node's refresh window +/// width (`RefreshWindow::computed_from`, the rule the installed plan +/// refreshes it on), or a query-time `Scan`'s read, the /// most offset + range its consumers read ([`read_back_ms`] without a span; /// `None`: it reads what it retains). A materialized node keeps what its /// `retention_ms` ($h_v$) covers: a tumbling-window chain its windows in -/// it, a query-time `Scan` its raw rows in it, any other view materialized -/// at ingestion time one output per refresh in it (one second of rows -/// when it maintains no window). The formulas' CPU is scaled by +/// it, a query-time `Scan` and raw rows materialized at ingestion time (a +/// `Scan`, or a `TimeShift` by an offset over one) their raw rows in it, +/// any other view materialized at ingestion time one output per refresh in +/// it (one second of rows when it maintains no window). The formulas' CPU is scaled by /// `settings.frozen_calibration_factor` (1 when unset). /// /// [`executions_per_second`]: crate::plan::execution_demand::executions_per_second @@ -286,14 +283,7 @@ pub fn node_resource_usage_and_cost<'a>( // windows; any other view materialized at ingestion time has one output // per refresh (per window it maintains, `window_ms`; per second of // arriving rows without one). - let output_interval_ms = match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => *tumbling_window_width_ms, - _ => window_ms.unwrap_or(ONE_SECOND_MS), - } - .max(1); + let output_interval_ms = output_interval_ms(&node.payload, window_ms); // A view materialized across evaluations keeps what its retention // covers. Without one (a reader reads it unbounded: a negative offset, // an `@` anchor) nothing bounds what it keeps and it cannot be @@ -307,13 +297,17 @@ pub fn node_resource_usage_and_cost<'a>( .max(1); let inputs: Vec = producers.iter().map(|p| p.output).collect(); // The tumbling windows its rows span: those a `TimeRange` selects from - // a chain, 1 otherwise. + // a chain (through the operators that keep each row's window, which + // pass them on: `Operator::keeps_input_window`), 1 otherwise. let tumbling_windows = match &node.payload { Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) - if chain_width_ms(producers).is_some() => + if chain_tumbling_window_width_ms(producers).is_some() => { selected_tumbling_windows(range, producers)? } + payload if payload.keeps_input_window() => producers + .first() + .map_or(1, |producer| producer.tumbling_windows), _ => 1, }; let sources: BTreeSet<&str> = match &node.payload { @@ -328,7 +322,39 @@ pub fn node_resource_usage_and_cost<'a>( // The rows per ms its sources receive, which its scans and time // ranges read, when the data declares an arrival rate. let rows_per_ms = source_rows_per_ms(&sources, data); + // Raw rows materialized at ingestion time: a `Scan`, or a `TimeShift` + // by an offset over one, across evaluations. It is a time-indexed store + // its consumers read back in, retaining `retention_ms` ($h_v$) of rows, + // which behind an offset includes the offset. + let raw_rows_materialized_at_ingestion = + node.keeps_raw_rows_at_ingestion(producers.first().map(|producer| producer.payload)); + // The raw rows of its sources back `lookback_ms`: (by arrival rate, + // when known; the rows they hold, when declared). + let raw_rows = |lookback_ms: Option| -> (Option, Option) { + let by_rate = rows_per_ms + .zip(lookback_ms) + .map(|(rows_per_ms, lookback_ms)| (rows_per_ms * lookback_ms as f64).round() as u64); + let rows_held: Option = sources + .iter() + .map(|source| data.sources.get(*source).and_then(|facts| facts.rows_held)) + .sum(); + (by_rate, rows_held) + }; + // The raw rows of its sources back `lookback_ms` (every row held + // without one), which a scan reads or keeps: by arrival rate, at most + // the rows they hold. + let raw_rows_back = |lookback_ms: Option| -> Option { + match raw_rows(lookback_ms) { + (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), + (by_rate, rows_held) => by_rate.or(rows_held), + } + .map(|rows| rows.max(1)) + }; // The fact a rate needs, named when it is missing. + let retained_without_a_bound = |source: &str| AnalyticalCostError::MissingSourceFact { + source_name: source.to_string(), + missing: "its rows_held (raw data retained without a bound)", + }; let missing_rate = |source: &str| AnalyticalCostError::MissingSourceFact { source_name: source.to_string(), missing: "an arrival rate (the arrival's rows_per_second, or the \ @@ -445,7 +471,6 @@ pub fn node_resource_usage_and_cost<'a>( // it, at most the rows the source holds (all of them // without a window). let source = source_name(source); - let rows_held = data.sources.get(source).and_then(|facts| facts.rows_held); // At ingestion, one second of rows; at query time, one // execution reads the most its consumers read (`window_ms`), // else what the scan retains (its `retention_ms`, $h_v$); @@ -454,27 +479,16 @@ pub fn node_resource_usage_and_cost<'a>( true => Some(ONE_SECOND_MS), false => window_ms.or(node.retention_ms), }; - let by_rate = |lookback: Option| { - rows_per_ms.zip(lookback).map(|(rows_per_ms, lookback)| { - (rows_per_ms * lookback as f64).round() as u64 - }) - }; // The raw rows a query-time scan, or an ingestion-time scan // materialized across evaluations, keeps: back to the // oldest row read, its `retention_ms` ($h_v$; an offset // window keeps more than it reads), at most those held. - if !ingestion || matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) { - let retained_rows = match (by_rate(node.retention_ms), rows_held) { - (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), - (by_rate, rows_held) => by_rate.or(rows_held), - }; - raw_retained_rows = retained_rows.map(|rows| rows.max(1)); + if !ingestion || raw_rows_materialized_at_ingestion { + raw_retained_rows = raw_rows_back(node.retention_ms); } - let by_rate = by_rate(lookback); - let rows = match (ingestion, by_rate, rows_held) { - (true, by_rate, _) => by_rate.ok_or_else(|| missing_rate(source))?, - (false, Some(by_rate), Some(rows_held)) => by_rate.min(rows_held), - (false, by_rate, rows_held) => by_rate.or(rows_held).ok_or_else(|| { + let rows = match ingestion { + true => raw_rows(lookback).0.ok_or_else(|| missing_rate(source))?, + false => raw_rows_back(lookback).ok_or_else(|| { AnalyticalCostError::MissingSourceFact { source_name: source.to_string(), missing: match lookback { @@ -502,14 +516,9 @@ pub fn node_resource_usage_and_cost<'a>( ); // Raw data materialized at ingestion time: its consumers // read the rows it retains, not one second of them. - let out = match (ingestion, node.lifetime) { - (true, Lifetime::AcrossEvaluations(_)) => edge(raw_retained_rows.ok_or( - AnalyticalCostError::MissingSourceFact { - source_name: source.to_string(), - missing: "its rows_held (raw data retained without a bound)", - }, - )?)?, - _ => out, + let out = match raw_rows_materialized_at_ingestion { + true => edge(raw_retained_rows.ok_or(retained_without_a_bound(source))?)?, + false => out, }; (out, estimate, format!("scan {} samples", out.rows)) } @@ -602,17 +611,33 @@ pub fn node_resource_usage_and_cost<'a>( ); (out, estimate, format!("limit to {} rows", out.rows)) } - NonASAPOp::TimeShift { .. } => ( - input, - estimate_operator( - PhysicalOperator::TimeShift, - OperatorStatistics::PassThrough { - edges: unary(input), - }, - ), - format!("time shift: pass {} rows, free", input.rows), - ), - NonASAPOp::TimeRange { range, .. } if chain_width_ms(producers).is_some() => { + NonASAPOp::TimeShift { .. } => { + // Raw rows materialized at ingestion time: like a + // materialized `Scan`'s, its consumers read the rows it + // retains (its retention, its offset included), not one + // second of them. + let out = match raw_rows_materialized_at_ingestion { + true => { + raw_retained_rows = raw_rows_back(node.retention_ms); + let source = sources.first().copied().unwrap_or("(none)"); + edge(raw_retained_rows.ok_or(retained_without_a_bound(source))?)? + } + false => input, + }; + ( + out, + estimate_operator( + PhysicalOperator::TimeShift, + OperatorStatistics::PassThrough { + edges: unary(input), + }, + ), + format!("time shift: pass {} rows, free", input.rows), + ) + } + NonASAPOp::TimeRange { range, .. } + if chain_tumbling_window_width_ms(producers).is_some() => + { // Over a chain, the windows inside the range: one // window's states per window. let windows = tumbling_windows; @@ -929,8 +954,8 @@ pub fn node_resource_usage_and_cost<'a>( } (0, medium) } - // Raw data materialized at ingestion time: the rows it retains. - (true, Lifetime::AcrossEvaluations(medium), Payload::NonASAP(NonASAPOp::Scan { .. })) => { + // Raw rows materialized at ingestion time: the rows it retains. + (_, Lifetime::AcrossEvaluations(medium), _) if raw_rows_materialized_at_ingestion => { (edge(raw_retained_rows.unwrap_or(out.rows))?.bytes, medium) } // Any other view materialized at ingestion time, a chain included: @@ -997,12 +1022,20 @@ pub fn node_resource_usage_and_cost<'a>( if cost.is_some_and(|cost| !cost.is_finite()) { return Err(AnalyticalCostError::Overflow); } + let chain_tumbling_window_width_ms = + node.payload.chain_tumbling_window_width_ms().or_else(|| { + node.payload + .keeps_input_window() + .then(|| chain_tumbling_window_width_ms(producers)) + .flatten() + }); Ok(NodeResourceUsageAndCost { output: out, sources, family, unchanged_columns, tumbling_windows, + chain_tumbling_window_width_ms, work_ms_per_execution: node_ms, cpu_ops_per_second: estimate.cpu_ops() * per_second, scanned_bytes_per_second: estimate.scan_bytes() as f64 * per_second, @@ -1036,16 +1069,11 @@ pub(crate) fn cost_source(per_node: &BTreeMap) -> } } -/// The tumbling-window width of the chain `producers` starts with, if it -/// is one: a `TimeRange` over it selects windows. -fn chain_width_ms(producers: &[ProducerOutput<'_>]) -> Option { - match producers.first()?.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(*tumbling_window_width_ms), - _ => None, - } +/// The tumbling-window width of the chain the first of `producers` reads +/// from, if any ([`ProducerOutput::chain_tumbling_window_width_ms`]): a +/// `TimeRange` over it selects windows. +fn chain_tumbling_window_width_ms(producers: &[ProducerOutput<'_>]) -> Option { + producers.first()?.chain_tumbling_window_width_ms } /// The windows a `TimeRange` of `range` selects from the chain @@ -1054,7 +1082,9 @@ fn selected_tumbling_windows( range: &std::time::Duration, producers: &[ProducerOutput<'_>], ) -> Result { - let width_ms = chain_width_ms(producers).unwrap_or(1).max(1); + let width_ms = chain_tumbling_window_width_ms(producers) + .unwrap_or(1) + .max(1); Ok(u64::try_from(range.as_millis() / u128::from(width_ms)) .map_err(|_| AnalyticalCostError::Overflow)? .max(1)) diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 1955c1e6b..98fadfad6 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -58,6 +58,7 @@ fn a_merge_over_a_chain_s_windows_outputs_one_state_per_group() { family: StateFamily::Exact(&ExactKind::Sum), unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: Some(60_000), }; let windows = node(Payload::NonASAP(NonASAPOp::TimeRange { range: Duration::from_secs(300), @@ -94,6 +95,94 @@ fn a_merge_over_a_chain_s_windows_outputs_one_state_per_group() { ); } +/// o11y q21's shape, `sum(rate(m[5m] offset 1h))` over a chain of 60 s +/// windows: a `TimeShift` by 1 h between the chain and the 5m +/// `TimeRange`, or above the `TimeRange`, changes nothing; nor does a +/// `Filter` or a `Project`, which keep each row's window +/// (`Operator::keeps_input_window`). The range selects five windows of +/// 1000 states, and the merge over them reads 5000 states into 1000, as +/// with nothing between (it read 1000, one window). +#[test] +fn an_operator_keeping_its_input_window_over_a_chain_or_its_windows_selects_what_the_range_does() { + use asap_types::ir::operator::TimeShift; + use asap_types::ir::scalar::{Predicate, ProjectItem, ScalarExpr, ScalarValue}; + let settings = CostInputs::default(); + let data = DataWorkload::default(); + let chain = node(Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + })); + let shift = node(Payload::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: 0, + })); + let filter = node(Payload::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + child: 0, + })); + let project = node(Payload::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: None, + expr: ScalarExpr::Column(0), + }], + child: 0, + })); + let range = node(Payload::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: 0, + })); + let merge = node(Payload::ASAP(ASAPOp::SummaryMerge { + children: vec![0], + group_by: None, + })); + // The merge's cost detail and output over `path`, costed bottom-up + // from the chain's one window of 1000 states. + let merge_over = |path: &[&PhysicalASAPDAGNode]| -> (String, EdgeStatistics) { + let mut below = ProducerOutput { + payload: &chain.payload, + output_schema: &chain.output_schema, + output: EdgeStatistics { + rows: 1_000, + bytes: 16_000, + }, + sources: BTreeSet::new(), + family: StateFamily::Exact(&ExactKind::Sum), + unchanged_columns: BTreeSet::new(), + tumbling_windows: 1, + chain_tumbling_window_width_ms: Some(60_000), + }; + for &step in path { + let usage = + node_resource_usage_and_cost(step, &[below], (1, 1.0), None, &data, &settings) + .unwrap(); + below = produced(step, &usage); + } + let usage = + node_resource_usage_and_cost(&merge, &[below], (1, 1.0), None, &data, &settings) + .unwrap(); + (usage.cost.detail, usage.output) + }; + let direct = merge_over(&[&range]); + assert!( + direct.0.starts_with("merge 5000 states into 1000;"), + "{}", + direct.0 + ); + for keeping in [&shift, &filter, &project] { + for path in [[keeping, &range], [&range, keeping]] { + assert_eq!(merge_over(&path), direct, "{:?}", keeping.payload); + } + } +} + /// A query-time node holds its state only /// while it computes: bytes × its work per execution × its executions /// per second, with no query latency. A KLL state of 2 MB whose own @@ -120,6 +209,7 @@ fn query_time_memory_is_held_only_while_the_node_computes() { family: StateFamily::Exact(&ExactKind::Sum), unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; let merge = PhysicalASAPDAGNode { lifetime: Lifetime::OneEvaluation(StorageMedium::Memory), @@ -218,17 +308,21 @@ fn a_query_time_scan_reads_what_its_consumers_read_and_keeps_its_retention() { assert_eq!(kept_bytes, usage.output.bytes * 3_601); } -/// An ingestion-time view of raw rows with no window below it (a -/// `TimeShift` by 90 s over the arriving rows) keeps what its +/// Raw rows materialized at ingestion time behind an offset (a +/// `TimeShift` by 90 s over the arriving rows) keep what their /// `retention_ms` covers, its offset included (a 5-minute read is retained -/// for 390 s, [`read_back_ms`]): one second of rows per second, and the -/// second being built. Over 1,000 rows per second of 64 bytes, a 390 s -/// retention keeps 64,000 × 391 bytes, not one second's 64,000 × 2; a -/// 690 s one keeps 64,000 × 691. +/// for 390 s, [`read_back_ms`]), at the source's arrival rate: 1,000 rows +/// per second retained 390 s keep 390,000 rows, not one second's 1,000; a +/// 690 s retention keeps 690,000. #[test] fn an_ingestion_time_view_of_raw_rows_keeps_the_rows_its_retention_covers() { + use asap_types::ir::schema::{DataType, Field}; + use asap_types::workload::DurationMs; let settings = CostInputs::default(); - let data = DataWorkload::default(); + let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); + for facts in data.sources.values_mut() { + facts.series_count = Some(1_000); + } let scan = node(Payload::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric: "m".into() }, predicates: vec![], @@ -241,16 +335,22 @@ fn an_ingestion_time_view_of_raw_rows_keeps_the_rows_its_retention_covers() { rows: 1_000, bytes: 64_000, }, - sources: BTreeSet::new(), + sources: ["m"].into(), family: StateFamily::None, unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; + let row_schema = Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]); let kept_bytes = |retention_ms: u64| { let shift = PhysicalASAPDAGNode { output_state: ExecutionDataState::INGESTION_ROWS, lifetime: Lifetime::AcrossEvaluations(StorageMedium::Memory), retention_ms: Some(retention_ms), + output_schema: row_schema.clone(), ..node(Payload::NonASAP(NonASAPOp::TimeShift { shift: asap_types::ir::operator::arguments::TimeShift { offset_ms: 90_000, @@ -270,8 +370,10 @@ fn an_ingestion_time_view_of_raw_rows_keeps_the_rows_its_retention_covers() { .unwrap() .peak_memory_bytes }; - assert_eq!(kept_bytes(390_000), 64_000 * 391); - assert_eq!(kept_bytes(690_000), 64_000 * 691); + let row_width_bytes = row_bytes(&row_schema); + assert!(row_width_bytes > 0); + assert_eq!(kept_bytes(390_000), 390_000 * row_width_bytes); + assert_eq!(kept_bytes(690_000), 690_000 * row_width_bytes); } /// A view materialized across evaluations without a retention is read @@ -299,6 +401,7 @@ fn a_view_without_retention_is_not_costed() { family: StateFamily::None, unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; let shift = PhysicalASAPDAGNode { output_state: ExecutionDataState::INGESTION_ROWS, @@ -328,7 +431,8 @@ fn a_view_without_retention_is_not_costed() { /// A view over an ingestion-time chain of 60 s windows is refreshed once /// per chain window, not once per second: `TimeShift(1h)` over the chain -/// maintains the chain's 60 s window ([`maintained_window_ms`]), and, +/// is refreshed on the chain's 60 s window (its refresh window, +/// `RefreshWindow::computed_from`, is the chain's), and, /// retained for 1 h 5 m (its 5-minute reader and its offset), keeps 65 /// windows' outputs and the one being built, as many as the chain below /// it, not 3,901 seconds' worth. @@ -357,12 +461,7 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { child: 0, })) }; - let operator_of = |id: u32| match id { - 0 => &chain.payload, - _ => &shift.payload, - }; - let window_ms = maintained_window_ms(1, operator_of, |id| (id == 1).then_some(0)); - assert_eq!(window_ms, Some(60_000)); + let window_ms = Some(60_000); let one_window_of_states = ProducerOutput { payload: &chain.payload, output_schema: &chain.output_schema, @@ -374,6 +473,7 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { family: StateFamily::Exact(&ExactKind::Sum), unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: Some(60_000), }; let usage = node_resource_usage_and_cost( &shift, diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 473aeec90..47922c71c 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -1,7 +1,7 @@ //! Costing a physical plan DAG node by node: each node's resource usage //! ([`node_resource_usage_and_cost`]), then the plan's usage, latency and //! cost ([`plan_cost`]). Stage 3 costs its candidates with the same -//! function and rules (`executions_per_second`, `maintained_window_ms`, +//! function and rules (`executions_per_second`, `RefreshWindow::computed_from`, //! `read_back_ms`, [`raw_data_disk_cost_per_second`], //! [`request_overhead_cost_per_second`]), so its objective is the //! `plan_cost` of the plan it selects. @@ -11,7 +11,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::node::analytical::AnalyticalCostError; use crate::node::operator_statistics::EdgeStatistics; use crate::node::state_bytes::{sketch_shape, StateInputRows}; -use crate::node::usage::{cost_source, maintained_window_ms}; +use crate::node::usage::{cost_source, reads_one_batch_of}; use crate::plan::execution_demand::{computations, evaluation_rate, query_starts}; use crate::plan::read_back::read_back_ms; use crate::{lookup, StateFamily}; @@ -27,6 +27,7 @@ use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; +use asap_types::stages::Lifetime; use asap_types::summary_type::SketchParams; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; @@ -104,24 +105,49 @@ pub fn plan_cost( // usage and cost ([`node_resource_usage_and_cost`]), children first. let multipliers = computations(dag, entries, settings.non_repeating_query_horizon_seconds) .map_err(|error| (first, error))?; - let mut producers_of: HashMap> = HashMap::new(); - for e in &dag.edges { - producers_of.entry(e.consumer).or_default().push(e.producer); - } + // Each node's producers in its payload's input order, whatever order + // the DAG lists its edges in (`PhysicalASAPDAGNode::producers`): its + // first producer is its child. + let producers_of: HashMap> = dag + .nodes + .iter() + .map(|node| (node.id, node.producers())) + .collect(); + let first_input_of = |id: PhysicalASAPNodeId| { + let input = producers_of[&id].first()?; + Some(&nodes[input].payload) + }; + // Each node keeping raw rows at ingestion, not stored: what an + // ingestion-time reader of it reads, one batch + // ([`reads_one_batch_of`]). + let one_batch_nodes: HashMap = dag + .nodes + .iter() + .filter(|node| node.keeps_raw_rows_at_ingestion(first_input_of(node.id))) + .map(|node| { + let mut one_batch = node.clone(); + one_batch.lifetime = Lifetime::None; + one_batch.retention_ms = None; + (node.id, one_batch) + }) + .collect(); let mut costed: HashMap> = HashMap::new(); + let mut one_batch: HashMap> = HashMap::new(); for node in &dag.nodes { - let views: Vec> = producers_of - .get(&node.id) - .map_or(&[][..], Vec::as_slice) + let views: Vec> = producers_of[&node.id] .iter() - .map(|p| costed[p].producer_output(&nodes[p].payload, &nodes[p].output_schema)) + .map(|p| { + let read = match reads_one_batch_of(node, nodes[p], first_input_of(*p)) { + true => &one_batch[p], + false => &costed[p], + }; + read.producer_output(&nodes[p].payload, &nodes[p].output_schema) + }) .collect(); let window_ms = match node.output_state.timing { - ExecutionTiming::IngestionTime => maintained_window_ms( - node.id, - |id| &nodes[&id].payload, - |id| producers_of.get(&id)?.first().copied(), - ), + ExecutionTiming::IngestionTime => { + dag.refresh_window(node.id).map(|window| window.width_ms) + } ExecutionTiming::QueryTime => scan_read_ms(dag, node), }; let node_costed = node_resource_usage_and_cost( @@ -133,6 +159,18 @@ pub fn plan_cost( settings, ) .map_err(|error| (node.id, error))?; + if let Some(unstored) = one_batch_nodes.get(&node.id) { + let batch = node_resource_usage_and_cost( + unstored, + &views, + multipliers[&node.id], + window_ms, + data, + settings, + ) + .map_err(|error| (node.id, error))?; + one_batch.insert(node.id, batch); + } costed.insert(node.id, node_costed); } let mut output: HashMap = HashMap::new(); diff --git a/crates/cost/src/plan/execution_demand.rs b/crates/cost/src/plan/execution_demand/mod.rs similarity index 63% rename from crates/cost/src/plan/execution_demand.rs rename to crates/cost/src/plan/execution_demand/mod.rs index b939d8fa6..24d8efe55 100644 --- a/crates/cost/src/plan/execution_demand.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -8,9 +8,11 @@ //! to cost once per request, as `computations` does. use crate::node::analytical::AnalyticalCostError; +use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::Operator; use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::stages::Lifetime; +use asap_types::stages::{queries_of_context, query_reads, Lifetime}; use asap_types::workload::{QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -281,86 +283,180 @@ pub fn executions_per_second<'e>( ) } -/// What computes a not-materialized node once per its own execution -/// ([`consumers`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +/// What computes a node that is not materialized across evaluations once +/// per its own execution ([`consumers`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Consumer { /// Query `.0` (an operator or scalar root), at its own rate. Query(usize), - /// A materialized node, at that node's rate. + /// Query `query` below a `TimeShift` anchored with `at` (`@ end()` or + /// `@ start()`): once per request ([`context_executions_per_second`]). + QueryRequest { query: usize, at: AtModifier }, + /// A node materialized across evaluations, at that node's rate. Materialized(PhysicalASAPNodeId), } -/// The consumers of each not-materialized node of `dag` that any consumer -/// reaches ([`computations`]): the queries and the materialized nodes whose -/// walk down their inputs reaches it before meeting a materialized node, -/// each once; the queries first, in order, then the materialized nodes in -/// `dag`'s node order. -pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> { +/// The consumers of each node of `dag` not materialized across evaluations +/// that any consumer reaches ([`computations`]): the queries +/// ([`query_reads`](asap_types::stages::query_reads)) and the nodes materialized across evaluations whose +/// walk down their inputs reaches it before meeting such a node, each +/// once, sorted. A `OneEvaluation` node is computed within its consumers' +/// context, so a walk goes through it. +pub fn consumers( + dag: &PhysicalASAPDAG, + entries: &[QueryWorkloadEntry], +) -> HashMap> { let materialized_nodes: Vec = dag .nodes .iter() - .filter(|n| n.lifetime != Lifetime::None) + .filter(|n| matches!(n.lifetime, Lifetime::AcrossEvaluations(_))) .map(|n| n.id) .collect(); let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); - let inputs = |id: PhysicalASAPNodeId| { + let inputs = |id: PhysicalASAPNodeId| -> Vec { dag.edges .iter() - .filter(move |e| e.consumer == id) + .filter(|e| e.consumer == id) .map(|e| e.producer) + .collect() }; - let walks = query_starts(dag) - .into_iter() - .enumerate() - .map(|(query, start)| (Consumer::Query(query), start)) - .chain( - materialized_nodes - .iter() - .map(|&id| (Consumer::Materialized(id), inputs(id).collect())), - ); + let payload: HashMap> = dag + .nodes + .iter() + .map(|node| (node.id, &node.payload)) + .collect(); let mut consumers: HashMap<_, Vec> = HashMap::new(); - for (consumer, start) in walks { + let queries = query_reads( + &query_starts(dag), + inputs, + |id| { + payload + .get(&id) + .and_then(|operator| operator.request_anchor()) + }, + |id| is_materialized.contains(&id), + entries, + ); + for (id, reads) in queries { + consumers + .entry(id) + .or_default() + .extend(reads.into_iter().map(|read| match read { + (query, None) => Consumer::Query(query), + (query, Some(at)) => Consumer::QueryRequest { query, at }, + })); + } + // Each node materialized across evaluations, over the nodes it reads + // down to the next such node. + for &materialized in &materialized_nodes { let mut seen = HashSet::new(); - let mut stack: Vec = start; + let mut stack = inputs(materialized); while let Some(id) = stack.pop() { if is_materialized.contains(&id) || !seen.insert(id) { continue; } - consumers.entry(id).or_default().push(consumer); + consumers + .entry(id) + .or_default() + .push(Consumer::Materialized(materialized)); stack.extend(inputs(id)); } } + // Queries first, in order, then the materialized nodes. + for of_node in consumers.values_mut() { + of_node.sort(); + } consumers } +/// How many times per second a node runs in one execution context of the +/// queries `queries` (a node not materialized across evaluations, run once +/// per execution of the context; Stage 3 and [`computations`] both cost it +/// so): with no `anchor`, at every evaluation of theirs, a range query's +/// steps each one ([`evaluation_rate`]); below a `TimeShift` anchored with +/// `anchor`, once per request: below `@ end()` once per distinct request +/// end ([`evaluation_rate`] of their request schedules); below +/// `@ start()` that per span, summed over the distinct spans, as two +/// queries requested together with different spans need the input at two +/// start times, and a result not kept across evaluations serves only the +/// request it is computed for. +/// +/// Example: two range queries every 60 s at phase 0 with steps of 30 s, +/// spans 1 h and 2 m: below `@ start()` 2/60 per second (one run per +/// request of each); below `@ end()` both end at `k · 60 s`, 1/60 per +/// second. +pub fn context_executions_per_second<'e>( + anchor: Option, + queries: impl IntoIterator, + non_repeating_query_horizon_seconds: f64, +) -> Result { + let queries = queries.into_iter(); + match anchor { + // Requests of one span need the input at one start time each; + // different spans need it at different times of one request. + Some(AtModifier::Start) => { + // An instant query anchors `@ start()` where a span-0 range + // query does. + let mut by_span: BTreeMap> = BTreeMap::new(); + for entry in queries { + by_span + .entry(entry.range_span_ms()) + .or_default() + .push(&entry.recurrence); + } + by_span + .values() + .map(|recurrences| { + evaluation_rate( + recurrences.iter().map(|&recurrence| (recurrence, None)), + non_repeating_query_horizon_seconds, + ) + }) + .sum() + } + // Once per request end, however many steps the request evaluates. + Some(_) => evaluation_rate( + queries.map(|entry| (&entry.recurrence, None)), + non_repeating_query_horizon_seconds, + ), + None => evaluation_rate( + queries.map(|entry| (&entry.recurrence, entry.range_evaluation)), + non_repeating_query_horizon_seconds, + ), + } +} + /// How often each node of `dag` is computed: `(multiplier, per_second)` -/// (module doc, "Cost model"). A materialized node is computed once for all -/// its queries: once per second at ingestion time (materialized at ingestion time), at -/// the combined rate of the queries reaching it (`OneEvaluation`, -/// [`evaluation_rate`]), or once per request of those queries for a chain -/// materialized across evaluations, whose request builds only its newest tumbling window -/// however many range steps it evaluates ([`executions_per_second`]). -/// `entries[q]` is query `q`'s workload entry (its recurrence and range -/// evaluation). A not-materialized node is computed once per -/// consumer that reaches it ([`consumers`]), at that consumer's rate. A consumer is a query -/// (operator or scalar root), at its own rate, or a materialized node, at -/// that node's rate; a walk stops at the first materialized node it meets, -/// and a node read several times within one consumer's walk counts once for -/// it; its `multiplier` is that number of consumers. A node computed at -/// ingestion time, materialized or not, computes each arriving row once: -/// once per second, `(1, 1.0)`, however many ingestion-time views of -/// however many refresh windows read it. +/// (module doc, "Cost model"). A node materialized across evaluations is +/// computed once for all its queries: once per second at ingestion time, +/// or once per request of the queries reaching it at query time, a chain +/// building only its newest tumbling window however many range steps it +/// evaluates ([`executions_per_second`]). `entries[q]` is query `q`'s +/// workload entry (its recurrence and range evaluation). Any other node +/// (ℓ None or `OneEvaluation`) is computed once per execution context of +/// the consumers that reach it ([`consumers`]; the execution contexts of +/// the concepts doc, section 5). A consumer is a query (operator or scalar +/// root) or a node materialized across evaluations; a walk stops at the +/// first such node it meets, and a node read several times within one +/// consumer's walk counts once for it. The queries on one evaluation grid +/// ([`QueryWorkloadEntry::evaluation_grid_ms`]) are one context, and the requests of the +/// queries on one grid below a `TimeShift` anchored with `@ end()` (or +/// `@ start()`) another, each at [`context_executions_per_second`]; a +/// materialized node is a context of its own, at its rate. Its +/// `multiplier` is that number of contexts. A node computed at ingestion +/// time, materialized or not, computes each arriving row once: once per +/// second, `(1, 1.0)`, however many ingestion-time views of however many +/// refresh windows read it. pub fn computations( dag: &PhysicalASAPDAG, entries: &[QueryWorkloadEntry], non_repeating_query_horizon_seconds: f64, ) -> Result, AnalyticalCostError> { - // Each materialized node's timing and lifetime. + // Each node materialized across evaluations: its timing and lifetime. let materialized_nodes: HashMap = dag .nodes .iter() - .filter(|node| node.lifetime != Lifetime::None) + .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| (node.id, (node.output_state.timing, node.lifetime))) .collect(); let inputs = |id: PhysicalASAPNodeId| { @@ -398,25 +494,42 @@ pub fn computations( )?; rate_of_materialized.insert(id, rate); } - // Each query consumer's rate. - let mut query_rates = Vec::new(); - for query in 0..starts.len() { - query_rates.push(evaluation_rate( - entries - .get(query) - .map(|entry| (&entry.recurrence, entry.range_evaluation)), - non_repeating_query_horizon_seconds, - )?); - } let mut count: HashMap<_, (u64, f64)> = HashMap::new(); - for (id, of_node) in consumers(dag) { + for (id, of_node) in consumers(dag, entries) { let entry = count.entry(id).or_default(); + // The queries reaching it, per anchor. + let mut queries_by_anchor: BTreeMap, BTreeSet> = BTreeMap::new(); for consumer in of_node { - entry.0 += 1; - entry.1 += match consumer { - Consumer::Query(query) => query_rates[query], - Consumer::Materialized(node) => rate_of_materialized[&node], + let (query, anchor) = match consumer { + Consumer::Query(query) => (query, None), + Consumer::QueryRequest { query, at } => (query, Some(at)), + Consumer::Materialized(node) => { + entry.0 += 1; + entry.1 += rate_of_materialized[&node]; + continue; + } }; + queries_by_anchor.entry(anchor).or_default().insert(query); + } + // One context per (evaluation grid, anchor). + for (anchor, reaching) in queries_by_anchor { + let grids: BTreeSet = reaching + .iter() + .map(|&query| { + entries + .get(query) + .map_or(1, QueryWorkloadEntry::evaluation_grid_ms) + }) + .collect(); + for grid_ms in grids { + let queries = queries_of_context(grid_ms, reaching.iter().copied(), entries); + entry.0 += 1; + entry.1 += context_executions_per_second( + anchor, + queries.iter().filter_map(|&query| entries.get(query)), + non_repeating_query_horizon_seconds, + )?; + } } } Ok(dag @@ -436,187 +549,4 @@ pub fn computations( } #[cfg(test)] -mod tests { - use super::*; - use asap_types::workload::{DemandEstimate, Rate, RepetitionInterval, TimestampMs}; - - fn fixed(interval: u32, phase: u64) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval), - evaluation_phase: TimestampMs(phase), - }) - } - - fn close(actual: f64, expected: f64) { - assert!((actual - expected).abs() < 1e-12, "{actual} != {expected}"); - } - - // Equal phases share evaluations; separate phases add. - #[test] - fn chosen_readers_preserve_phase_accounting() { - let queries = [ - fixed(20_000, 5_000), - fixed(20_000, 25_000), - fixed(20_000, 0), - ]; - close( - evaluation_rate(queries[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), - 1.0 / 20.0, - ); - close( - evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), - 2.0 / 20.0, - ); - // No global subset cache: each caller's selected queries determine the rate. - close(evaluation_rate(std::iter::empty(), 3600.0).unwrap(), 0.0); - } - - // Range offsets union within one cadence; retained state instead uses requests. - #[test] - fn range_offsets_and_request_rates_stay_distinct() { - let recurrence = fixed(60_000, 0); - let ranges = [ - Some(RangeEvaluation { - step_ms: 60_000, - span_ms: 3_600_000, - }), - Some(RangeEvaluation { - step_ms: 30_000, - span_ms: 600_000, - }), - None, - ]; - close( - evaluation_rate(ranges.iter().map(|r| (&recurrence, *r)), 3600.0).unwrap(), - 71.0 / 60.0, - ); - close( - evaluation_rate(ranges.iter().map(|_| (&recurrence, None)), 3600.0).unwrap(), - 1.0 / 60.0, - ); - } - - // Different fixed intervals remain additive even if timestamps can coincide. - #[test] - fn different_intervals_preserve_existing_conservative_rate() { - let queries = [fixed(20_000, 0), fixed(30_000, 0)]; - close( - evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), - 1.0 / 20.0 + 1.0 / 30.0, - ); - } - - // Estimates add with range multiplicity; scheduled/one-off/unknown use a maximum. - #[test] - fn estimated_and_batched_nonrepeating_demand_keep_their_policy() { - let estimate = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(2.0), - observed_at: None, - valid_for: None, - })); - let scheduled = QueryRecurrence::Repeated(RepeatedDemand::Scheduled(vec![ - TimestampMs(0), - TimestampMs(10), - ])); - let once = QueryRecurrence::OneTime { - invocations: 4, - execute_at: None, - }; - let unknown = QueryRecurrence::Unknown; - let range = Some(RangeEvaluation { - step_ms: 1000, - span_ms: 2000, - }); - close( - evaluation_rate( - [ - (&estimate, range), - (&estimate, None), - (&scheduled, range), - (&once, None), - (&unknown, None), - ], - 100.0, - ) - .unwrap(), - 8.0 + 6.0 / 100.0, - ); - } - - #[test] - fn billion_offset_mixed_ranges_are_counted_arithmetically() { - let recurrence = fixed(1000, 0); - let rate = evaluation_rate( - [ - ( - &recurrence, - Some(RangeEvaluation { - span_ms: 1_000_000_000, - step_ms: 2, - }), - ), - ( - &recurrence, - Some(RangeEvaluation { - span_ms: 1_000_000_000, - step_ms: 3, - }), - ), - ], - 60.0, - ) - .unwrap(); - assert_eq!(rate, 666_666_668.0); - } - - #[test] - fn arithmetic_union_matches_small_explicit_offset_sets() { - for left_step in 1..=7 { - for right_step in 1..=7 { - for left_span in [0, 5, 17, 30] { - for right_span in [0, 9, 23, 30] { - let mut ranges = BTreeMap::new(); - for (step, span) in - [(left_step, left_span), (right_step, right_span), (11, 29)] - { - let longest = ranges.entry(step).or_insert(0u64); - *longest = (*longest).max(span); - } - let mut oracle = BTreeSet::new(); - for (&step, &span) in &ranges { - oracle.extend((0..=span / step).map(|k| k * step)); - } - assert_eq!( - distinct_instants_per_request(&ranges), - oracle.len() as u64, - "{ranges:?}" - ); - } - } - } - } - let large = 1u64 << 63; - assert_eq!( - offset_intersection_count(&[(large - 1, large), (large - 3, large)]), - 1 - ); - } - - // Invalid fixed intervals and estimated rates still produce the evaluator's error. - #[test] - fn invalid_recurrence_is_rejected() { - assert!(matches!( - evaluation_rate([(&fixed(0, 0), None)], 3600.0), - Err(AnalyticalCostError::InvalidRecurrence) - )); - let invalid = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(-1.0), - observed_at: None, - valid_for: None, - })); - assert!(matches!( - evaluation_rate([(&invalid, None)], 3600.0), - Err(AnalyticalCostError::InvalidRecurrence) - )); - } -} +mod tests; diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs new file mode 100644 index 000000000..832d15d11 --- /dev/null +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -0,0 +1,510 @@ +use super::*; +use asap_types::ir::NonASAPOp; +use asap_types::workload::{DemandEstimate, Rate, RepetitionInterval, TimestampMs}; + +fn fixed(interval: u32, phase: u64) -> QueryRecurrence { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval), + evaluation_phase: TimestampMs(phase), + }) +} + +fn close(actual: f64, expected: f64) { + assert!((actual - expected).abs() < 1e-12, "{actual} != {expected}"); +} + +// Equal phases share evaluations; separate phases add. +#[test] +fn chosen_readers_preserve_phase_accounting() { + let queries = [ + fixed(20_000, 5_000), + fixed(20_000, 25_000), + fixed(20_000, 0), + ]; + close( + evaluation_rate(queries[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), + 1.0 / 20.0, + ); + close( + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), + 2.0 / 20.0, + ); + // No global subset cache: each caller's selected queries determine the rate. + close(evaluation_rate(std::iter::empty(), 3600.0).unwrap(), 0.0); +} + +// Range offsets union within one cadence; retained state instead uses requests. +#[test] +fn range_offsets_and_request_rates_stay_distinct() { + let recurrence = fixed(60_000, 0); + let ranges = [ + Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }), + Some(RangeEvaluation { + step_ms: 30_000, + span_ms: 600_000, + }), + None, + ]; + close( + evaluation_rate(ranges.iter().map(|r| (&recurrence, *r)), 3600.0).unwrap(), + 71.0 / 60.0, + ); + close( + evaluation_rate(ranges.iter().map(|_| (&recurrence, None)), 3600.0).unwrap(), + 1.0 / 60.0, + ); +} + +// Different fixed intervals remain additive even if timestamps can coincide. +#[test] +fn different_intervals_preserve_existing_conservative_rate() { + let queries = [fixed(20_000, 0), fixed(30_000, 0)]; + close( + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), + 1.0 / 20.0 + 1.0 / 30.0, + ); +} + +// Estimates add with range multiplicity; scheduled/one-off/unknown use a maximum. +#[test] +fn estimated_and_batched_nonrepeating_demand_keep_their_policy() { + let estimate = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(2.0), + observed_at: None, + valid_for: None, + })); + let scheduled = QueryRecurrence::Repeated(RepeatedDemand::Scheduled(vec![ + TimestampMs(0), + TimestampMs(10), + ])); + let once = QueryRecurrence::OneTime { + invocations: 4, + execute_at: None, + }; + let unknown = QueryRecurrence::Unknown; + let range = Some(RangeEvaluation { + step_ms: 1000, + span_ms: 2000, + }); + close( + evaluation_rate( + [ + (&estimate, range), + (&estimate, None), + (&scheduled, range), + (&once, None), + (&unknown, None), + ], + 100.0, + ) + .unwrap(), + 8.0 + 6.0 / 100.0, + ); +} + +#[test] +fn billion_offset_mixed_ranges_are_counted_arithmetically() { + let recurrence = fixed(1000, 0); + let rate = evaluation_rate( + [ + ( + &recurrence, + Some(RangeEvaluation { + span_ms: 1_000_000_000, + step_ms: 2, + }), + ), + ( + &recurrence, + Some(RangeEvaluation { + span_ms: 1_000_000_000, + step_ms: 3, + }), + ), + ], + 60.0, + ) + .unwrap(); + assert_eq!(rate, 666_666_668.0); +} + +#[test] +fn arithmetic_union_matches_small_explicit_offset_sets() { + for left_step in 1..=7 { + for right_step in 1..=7 { + for left_span in [0, 5, 17, 30] { + for right_span in [0, 9, 23, 30] { + let mut ranges = BTreeMap::new(); + for (step, span) in [(left_step, left_span), (right_step, right_span), (11, 29)] + { + let longest = ranges.entry(step).or_insert(0u64); + *longest = (*longest).max(span); + } + let mut oracle = BTreeSet::new(); + for (&step, &span) in &ranges { + oracle.extend((0..=span / step).map(|k| k * step)); + } + assert_eq!( + distinct_instants_per_request(&ranges), + oracle.len() as u64, + "{ranges:?}" + ); + } + } + } + } + let large = 1u64 << 63; + assert_eq!( + offset_intersection_count(&[(large - 1, large), (large - 3, large)]), + 1 + ); +} + +/// One workload entry per recurrence, each with `range_evaluation`. +fn entries( + recurrence: &[QueryRecurrence], + range_evaluation: Option, +) -> Vec { + recurrence + .iter() + .map(|recurrence| QueryWorkloadEntry { + query: asap_types::workload::Query("q".into()), + requirements: Default::default(), + predictability: Default::default(), + recurrence: recurrence.clone(), + range_evaluation, + }) + .collect() +} + +/// A plan node `id` running `payload` with `timing` and `lifetime`. +fn node( + id: PhysicalASAPNodeId, + payload: Operator, + timing: ExecutionTiming, + lifetime: Lifetime, +) -> asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode { + use asap_types::ir::properties::{DataPrimitive, ExecutionDataState}; + use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; + asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode { + id, + output_state: ExecutionDataState { + timing, + primitive: DataPrimitive::of(&payload, asap_types::ir::OperatorResultKind::RangeVector), + }, + parameters: matches!(payload, Operator::ASAP(_)) + .then_some(asap_types::summary_type::SketchParams::Kll { k: 200 }), + payload, + lifetime, + output_schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::new("value", FieldDataType::Plain(DataType::Float64), false), + ], + 0, + vec![], + ), + retention_ms: None, + selected_physical_candidate: None, + view_coverage: None, + } +} + +fn scan() -> Operator { + use asap_types::ir::operator::Source; + use asap_types::ir::schema::{DataType, Field, Schema}; + Operator::NonASAP(asap_types::ir::NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +fn last(seconds: u64, child: PhysicalASAPNodeId) -> Operator { + Operator::NonASAP(asap_types::ir::NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(seconds), + kind: asap_types::ir::TimeRangeKind::Range, + child, + }) +} + +/// `m[5m]` read by three queries: two every 10 s at phase 0, one every +/// 30 s. The two on the 10 s grid are one execution context, which +/// computes the not-materialized range and scan once for both: two +/// contexts, 1/10 + 1/30 per second. +#[test] +fn a_not_materialized_node_is_computed_once_per_evaluation_grid() { + let query_time = ExecutionTiming::QueryTime; + let dag = PhysicalASAPDAG::new( + vec![ + node(0, scan(), query_time, Lifetime::None), + node(1, last(300, 0), query_time, Lifetime::None), + ], + vec![1, 1, 1], + vec![], + ) + .unwrap(); + let recurrence = [fixed(10_000, 0), fixed(10_000, 0), fixed(30_000, 0)]; + let rates = computations(&dag, &entries(&recurrence, None), 3600.0).unwrap(); + for id in [0, 1] { + assert_eq!(rates[&id].0, 2); + close(rates[&id].1, 1.0 / 10.0 + 1.0 / 30.0); + } +} + +/// `max_over_time(m[5m] @ end())` requested every 60 s over the last +/// hour at a 10 s step (361 steps per request): the shift runs at +/// every step, its input (the range and the scan) once per request, +/// at the request end: 1/60 per second, not 361/60. +#[test] +fn a_range_query_computes_its_at_end_subtree_once_per_request() { + use asap_types::ir::operator::TimeShift; + let query_time = ExecutionTiming::QueryTime; + let at_end = Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(AtModifier::End), + }, + child: 1, + }); + let dag = PhysicalASAPDAG::new( + vec![ + node(0, scan(), query_time, Lifetime::None), + node(1, last(300, 0), query_time, Lifetime::None), + node(2, at_end, query_time, Lifetime::None), + ], + vec![2], + vec![], + ) + .unwrap(); + let hour = RangeEvaluation { + step_ms: 10_000, + span_ms: 3_600_000, + }; + let rates = computations(&dag, &entries(&[fixed(60_000, 0)], Some(hour)), 3600.0).unwrap(); + close(rates[&2].1, 361.0 / 60.0); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 1.0 / 60.0); + } +} + +/// `m[5m]` below a `TimeShift` anchored with `at`, the shift a +/// `OneEvaluation` result read by every query (`roots`). +fn anchored_dag(at: AtModifier, roots: Vec) -> PhysicalASAPDAG { + use asap_types::ir::operator::TimeShift; + use asap_types::stages::StorageMedium; + let query_time = ExecutionTiming::QueryTime; + let shift = Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(at), + }, + child: 1, + }); + PhysicalASAPDAG::new( + vec![ + node(0, scan(), query_time, Lifetime::None), + node(1, last(300, 0), query_time, Lifetime::None), + node( + 2, + shift, + query_time, + Lifetime::OneEvaluation(StorageMedium::Memory), + ), + ], + roots, + vec![], + ) + .unwrap() +} + +/// Two range queries every 300 s at a 30 s step over 600 s (21 steps per +/// request) share one `OneEvaluation` `@ end()` shift: it runs at every +/// step of their one context, 21/300 per second, and its input once per +/// request, 1/300: a walk goes through a `OneEvaluation` node. +#[test] +fn a_shared_one_evaluation_at_end_shift_runs_its_input_once_per_request() { + let span = RangeEvaluation { + step_ms: 30_000, + span_ms: 600_000, + }; + let recurrence = [fixed(300_000, 0), fixed(300_000, 0)]; + let rates = computations( + &anchored_dag(AtModifier::End, vec![2, 2]), + &entries(&recurrence, Some(span)), + 3600.0, + ) + .unwrap(); + assert_eq!(rates[&2].0, 1); + close(rates[&2].1, 21.0 / 300.0); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 1.0 / 300.0); + } +} + +/// Two range queries every 60 s at a 30 s step, over 1 h and over 90 s, +/// below one `@ start()` shift: one context (one evaluation grid, one +/// anchor), whose requests start at `k · 60 s` and `k · 60 s + 30 s`: the +/// input runs 2/60 per second. Over 1 h and over 2 m, one request needs +/// the input at two start times, its end less 1 h and its end less 2 m: +/// still 2/60. The same query twice runs once, and so does one schedule +/// written at phases 10 s and 70 s. +#[test] +fn at_start_inputs_run_once_per_request_of_each_span() { + let with_span = |span_ms| QueryWorkloadEntry { + query: asap_types::workload::Query("q".into()), + requirements: Default::default(), + predictability: Default::default(), + recurrence: fixed(60_000, 0), + range_evaluation: Some(RangeEvaluation { + step_ms: 30_000, + span_ms, + }), + }; + let entries = [with_span(3_600_000), with_span(90_000)]; + let rates = computations( + &anchored_dag(AtModifier::Start, vec![2, 2]), + &entries, + 3600.0, + ) + .unwrap(); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 2.0 / 60.0); + } + close( + context_executions_per_second(Some(AtModifier::End), &entries, 3600.0).unwrap(), + 1.0 / 60.0, + ); + let same_start = [with_span(3_600_000), with_span(120_000)]; + close( + context_executions_per_second(Some(AtModifier::Start), &same_start, 3600.0).unwrap(), + 2.0 / 60.0, + ); + let twice = [with_span(120_000), with_span(120_000)]; + close( + context_executions_per_second(Some(AtModifier::Start), &twice, 3600.0).unwrap(), + 1.0 / 60.0, + ); + let mut phase_70s = with_span(120_000); + phase_70s.recurrence = fixed(60_000, 70_000); + let mut phase_10s = with_span(120_000); + phase_10s.recurrence = fixed(60_000, 10_000); + close( + context_executions_per_second(Some(AtModifier::Start), &[phase_10s, phase_70s], 3600.0) + .unwrap(), + 1.0 / 60.0, + ); +} + +// Invalid fixed intervals and estimated rates still produce the evaluator's error. +#[test] +fn invalid_recurrence_is_rejected() { + assert!(matches!( + evaluation_rate([(&fixed(0, 0), None)], 3600.0), + Err(AnalyticalCostError::InvalidRecurrence) + )); + let invalid = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(-1.0), + observed_at: None, + valid_for: None, + })); + assert!(matches!( + evaluation_rate([(&invalid, None)], 3600.0), + Err(AnalyticalCostError::InvalidRecurrence) + )); +} + +/// `m[5m]` below an `@ end()` shift read by an instant query every 60 s, +/// and read directly by another instant query every 60 s: the anchor is +/// the evaluation time, so the range and the scan are computed in the one +/// evaluation context, once per minute. +#[test] +fn an_instant_at_end_input_runs_in_the_evaluation_context() { + let rates = computations( + &anchored_dag(AtModifier::End, vec![2, 1]), + &entries(&[fixed(60_000, 0), fixed(60_000, 0)], None), + 3600.0, + ) + .unwrap(); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 1.0 / 60.0); + } +} + +/// An ingestion-time chain of 60 s windows over `m` (1,000 rows per +/// second), read back 5 minutes at query time every 60 s, reading either +/// a `Scan` that is not materialized or raw rows `m` kept at ingestion for +/// an hour: either way it folds in one arriving batch per second +/// (`reads_one_batch_of`), so `plan_cost` charges it the same, not as if +/// it read the hour of stored rows. +#[test] +fn an_ingestion_time_reader_of_raw_rows_kept_at_ingestion_reads_one_batch() { + use asap_types::ir::operator::Reduction; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::FieldDataType; + use asap_types::stages::StorageMedium; + use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; + use asap_types::workload::{DataWorkload, DurationMs}; + let ingestion = ExecutionTiming::IngestionTime; + let chain = Operator::ASAP(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }); + let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); + for facts in data.sources.values_mut() { + facts.series_count = Some(1_000); + } + let chain_cost = |scan_lifetime: Lifetime, scan_retention_ms: Option| { + let mut stored_chain = node( + 1, + chain.clone(), + ingestion, + Lifetime::AcrossEvaluations(StorageMedium::Memory), + ); + stored_chain.retention_ms = Some(300_000); + let mut raw = node(0, scan(), ingestion, scan_lifetime); + raw.retention_ms = scan_retention_ms; + let dag = PhysicalASAPDAG::new( + vec![ + raw, + stored_chain, + node(2, last(300, 1), ExecutionTiming::QueryTime, Lifetime::None), + ], + vec![2], + vec![], + ) + .unwrap(); + let (cost, _) = crate::plan_cost( + &dag, + &entries(&[fixed(60_000, 0)], None), + &data, + &crate::CostInputs::default(), + ) + .unwrap(); + cost.per_node[&1].cost_per_second.unwrap() + }; + let over_arriving_rows = chain_cost(Lifetime::None, None); + let over_kept_rows = chain_cost( + Lifetime::AcrossEvaluations(StorageMedium::Memory), + Some(3_600_000), + ); + close(over_kept_rows, over_arriving_rows); +} diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 81e58ace6..51100e33f 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -57,6 +57,16 @@ fn selected_plan_objective_and_plan_cost( let models = PlanningModels::builtin() .with_cost_inputs(settings) .with_deployment_capabilities(&capabilities); + selected_plan_objective_and_plan_cost_with(workload, models, settings) +} + +/// The plan Stage 3 selects for `workload` with `models`, whose cost +/// inputs are `settings`, with the MILP's optimum and its `plan_cost`. +fn selected_plan_objective_and_plan_cost_with( + workload: &PlanningWorkload, + models: PlanningModels<'_>, + settings: &CostInputs, +) -> (PhysicalASAPDAG, (f64, f64)) { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap(); let roots = lower_promql_roots(workload).unwrap(); @@ -230,3 +240,397 @@ fn raw_data_on_disk_costing_less_than_the_solver_tolerance_still_steers_selectio "selected {plan_cost} is not cheaper than {ignoring_disk_cost}, the plan selected ignoring disk" ); } + +/// Two windowed quantiles over one metric (1 m and 5 m, every 10 s; 10,000 +/// series every 15 s), accuracy not enforced: the selected plan shares one +/// query-time scan, whose execution reads the 5 m window once and retains +/// it ($h_v$, constraint (R)), at most 0.144/s. +#[test] +fn the_objective_is_the_plan_cost_on_a_shared_scan() { + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + every_10s("quantile_over_time(0.5, data[1m])"), + every_10s("quantile_over_time(0.9, data[5m])"), + ], + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 666.67}}, + "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}, + }, + })) + .unwrap(); + let models = PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + let (_, (objective, plan_cost)) = + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()); + assert_equal("shared scan", (objective, plan_cost)); + assert!(plan_cost <= 0.144, "{plan_cost}"); +} + +/// The o11y benchmark's q21: `sum(rate(..[5m] offset 1h))`, a 1 h range +/// query every 60 s. Raw rows it reads may be materialized at ingestion +/// time behind the offset (a `TimeShift` over the `Scan`); that store +/// keeps 2 h 5 m of rows (its retention: the 1 h span, the 5 m range and +/// the 1 h offset), which its `TimeRange` reads 5 m of, in the MILP and in +/// `plan_cost` alike (the MILP's objective was 0.326/s for a plan +/// `plan_cost` charged 0.0219/s, one second of rows retained). +#[test] +fn the_objective_is_the_plan_cost_on_an_offset_range_query() { + let source = serde_json::json!({"sample_interval": 1000, "series_count": 1000, "metric_type": "counter"}); + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [{ + "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m] offset 1h))", + "demand": {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, + "predictability": {"predictable": {"known_at": null}}, + "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}, + }], + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"http_requests_total": source}, + }, + })) + .unwrap(); + assert_equal( + "offset range query", + selected_plan_objective_and_plan_cost_with( + &w, + PlanningModels::builtin(), + &CostInputs::default(), + ) + .1, + ); +} + +/// A query `query` every 10 s at phase 0, within 1 % at δ = 0.01. +fn every_10s(query: &str) -> serde_json::Value { + serde_json::json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": 10000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.01}}, + "predictability": {"predictable": {"known_at": null}}, + }) +} + +/// `query` requested every `interval_ms` at phase 0 as a range query with +/// steps of `step_ms` over `span_ms`, or as an instant query without one. +fn requested(query: &str, interval_ms: u32, range: Option<(u64, u64)>) -> serde_json::Value { + let mut entry = serde_json::json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": interval_ms, "evaluation_phase": 0}}, + "requirements": {"max_latency_ms": null}, + "predictability": {"predictable": {"known_at": null}}, + }); + if let Some((step_ms, span_ms)) = range { + entry["range_evaluation"] = serde_json::json!({"step_ms": step_ms, "span_ms": span_ms}); + } + entry +} + +/// `entries` over the gauge `m` (1,000 series, one sample per second). +fn over_m(entries: Vec) -> PlanningWorkload { + serde_json::from_value(serde_json::json!({ + "query_workload": {"language": "promql", "repeating_queries": entries}, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}}, + }, + })) + .unwrap() +} + +/// Two range queries every 300 s at 30 s steps over 600 s that read one +/// `@ end()` subtree: they share its shift, `OneEvaluation`, and its input +/// runs once per request, in the MILP and in `plan_cost` alike +/// (`plan_cost` charged it at every step: 0.1513/s for an objective of +/// 0.0473/s). +#[test] +fn the_objective_is_the_plan_cost_on_a_shared_at_end_subtree() { + let w = over_m(vec![ + requested( + "sum_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + requested( + "max_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + ]); + assert_equal( + "shared @ end() subtree", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} + +/// Two range queries every 60 s at 30 s steps, over 1 h and over 90 s, +/// below `@ start()`: one context, whose requests start at two times each +/// minute, `k · 60 s` and `k · 60 s + 30 s`; Stage 3 and `plan_cost` count +/// both. +#[test] +fn the_objective_is_the_plan_cost_on_at_start_subtrees_of_two_spans() { + let w = over_m(vec![ + requested( + "max_over_time(m[5m] @ start())", + 60_000, + Some((30_000, 3_600_000)), + ), + requested( + "max_over_time(m[5m] @ start())", + 60_000, + Some((30_000, 90_000)), + ), + ]); + assert_equal( + "@ start() subtrees of two spans", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} + +/// An instant `max_over_time(m[5m])` every 300 s and the range query +/// `max_over_time(m[5m] @ start())` every 300 s at 30 s steps over 600 s: +/// the range query reads `m[5m]` at its requests' start, 600 s before the +/// instant query's evaluation, so no result computed for one evaluation is +/// read by both (a `OneEvaluation` `m[5m]` served both). +#[test] +fn an_at_start_subtree_shares_no_evaluation_result_with_an_instant_query() { + let w = over_m(vec![ + requested("max_over_time(m[5m])", 300_000, None), + requested( + "max_over_time(m[5m] @ start())", + 300_000, + Some((30_000, 600_000)), + ), + ]); + let (plan, objective_and_plan_cost) = + selected_plan_objective_and_plan_cost(&w, &CostInputs::default(), true); + assert_equal("@ start() beside an instant query", objective_and_plan_cost); + let producers = |id| { + plan.edges + .iter() + .filter(move |edge| edge.consumer == id) + .map(|edge| edge.producer) + }; + // The nodes each query's walk reaches below its root, stopping at + // results kept across evaluations or computed at ingestion time. + let reached = |root: usize| { + let mut reached = std::collections::BTreeSet::new(); + let mut pending = vec![root]; + while let Some(id) = pending.pop() { + let node = &plan.nodes[id]; + if matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) + || node.output_state.timing == ExecutionTiming::IngestionTime + || !reached.insert(id) + { + continue; + } + pending.extend(producers(id)); + } + reached + }; + let (instant, at_start) = (reached(plan.roots[0]), reached(plan.roots[1])); + assert!( + instant.is_disjoint(&at_start), + "{instant:?} and {at_start:?} share a node: {plan:#?}" + ); +} + +/// `@ end()` and `@ start()` range queries every 300 s at 30 s steps next +/// to the plain `sum_over_time(m[5m])` every 30 s, on the same 30 s grid: +/// an anchored input runs once per request of the queries that reach its +/// shift, not of every query on the grid reaching `m[5m]` (the objective +/// was 0.2404/s for a plan `plan_cost` charged 0.1364/s). +#[test] +fn the_objective_is_the_plan_cost_on_anchored_and_plain_queries_on_one_grid() { + let w = over_m(vec![ + requested( + "sum_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + requested( + "max_over_time(m[5m] @ start())", + 300_000, + Some((30_000, 600_000)), + ), + requested("sum_over_time(m[5m])", 30_000, None), + ]); + assert_equal( + "anchored and plain queries on one grid", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} + +/// Example 3a of the planner-layering design doc: five one-off p99s over +/// 1,000,000 series sampled every 15 s, over the last 5 y, 1 y, 1 y offset +/// 1 y and 2 y, and 3 y offset 2 y, accuracy guarantees ignored. An +/// ingestion-time view behind an offset retains one output per window it +/// maintains, so its $\pi_v$ is one output's cost over that window: costed +/// per second instead, $\pi_v h_v$ of a view kept for years was 6.6e12/s, +/// its $W^{\mathrm{mat}}_v$ as negative, and the objective's scale left the +/// solver at a plan costing 4.68e7/s instead of 2.17e4/s. +#[test] +fn the_objective_is_the_plan_cost_on_one_off_queries_over_years() { + let query = |query: &str| { + serde_json::json!({ + "query": query, + "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.005, "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, + "predictability": "ad_hoc", "invocations": 1, "execute_at": 1767225600000_u64, + }) + }; + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": {"language": "promql", "query_batch": [ + query("quantile_over_time(0.99, latency_ms[5y])"), + query("quantile_over_time(0.99, latency_ms[1y])"), + query("quantile_over_time(0.99, latency_ms[1y] offset 1y)"), + query("quantile_over_time(0.99, latency_ms[1y] offset 2y)"), + query("quantile_over_time(0.99, latency_ms[3y] offset 2y)"), + ]}, + "data_workload": { + "arrival": {"mixed": {"rows_per_second": 1_000_000.0 / 15.0}}, + "sources": {"latency_ms": {"series_count": 1_000_000, "sample_interval": 15_000}}, + }, + })) + .unwrap(); + let models = PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + let (objective, plan_cost) = + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1; + assert_equal("one-off queries over years", (objective, plan_cost)); + assert!(objective < 30_000.0, "objective {objective}"); +} + +/// `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, both instant +/// every 60 s: an instant query's `@ end()` is its evaluation time, so the +/// anchored input runs in the evaluation context and shares `m[5m]` with +/// the plain query, in the MILP and in `plan_cost` alike (0.0360/s; in a +/// request context of its own it cost 0.0620/s). +#[test] +fn an_instant_at_end_input_shares_the_evaluation_context() { + let w = over_m(vec![ + requested("sum_over_time(m[5m] @ end())", 60_000, None), + requested("sum_over_time(m[5m])", 60_000, None), + ]); + let (objective, plan_cost) = objective_and_plan_cost(&w, &CostInputs::default(), true); + assert_equal( + "instant @ end() beside a plain query", + (objective, plan_cost), + ); + assert!(objective < 0.04, "objective {objective}"); +} + +/// `sum_over_time(m[1m])` every 20 s at phase 10 s and +/// `max_over_time(m[1m])` every 10 s: one 10 s grid, two phases. Stage 3 +/// and `plan_cost` pick the queries of each context with one rule +/// (`queries_of_context`), and the optimum is the plan's cost. +#[test] +fn the_objective_is_the_plan_cost_on_two_phases_of_one_grid() { + let mut late = requested("sum_over_time(m[1m])", 20_000, None); + late["demand"]["fixed_interval_at"]["evaluation_phase"] = serde_json::json!(10_000); + let w = over_m(vec![late, requested("max_over_time(m[1m])", 10_000, None)]); + assert_equal( + "two phases of one grid", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} + +/// An instant `sum_over_time(m[5m] @ end())` and a range +/// `sum_over_time(m[5m] @ end() offset 10m)` over the last 10 minutes, +/// both every 60 s: `m[5m]` is read by the instant query in its +/// evaluation context and by the range query once per request below its +/// shift, so a run in the evaluation context is rated by the instant query +/// only, in the MILP and in `plan_cost` alike (the objective was 0.3642/s +/// for a plan `plan_cost` charged 0.1542/s). +#[test] +fn the_objective_is_the_plan_cost_on_an_instant_and_a_shifted_range_at_end() { + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": {"language": "promql", "repeating_queries": [ + requested("sum_over_time(m[5m] @ end())", 60_000, None), + requested( + "sum_over_time(m[5m] @ end() offset 600s)", + 60_000, + Some((60_000, 600_000)), + ), + ]}, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 2000.0}}, + "sources": { + "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, + "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + }, + }, + })) + .unwrap(); + let capabilities = DeploymentCapabilities { + supported_summaries: Some(asap_executor::supported_summary_families()), + ..Default::default() + }; + let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); + assert_equal( + "instant and shifted range @ end()", + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1, + ); +} + +/// Two range queries every 300 s on one grid reach one `@ end()` group, +/// `m[5m] @ end()`, and may read it through different candidates: a sum +/// over 10 minutes at 30 s steps and a median over an hour at 60 s steps, +/// accuracy guarantees ignored (so the median may read a sketch). Stage 3 +/// rates the anchored context from the memo, `plan_cost` from the plan's +/// walk; the optimum is the plan's cost. +#[test] +fn the_objective_is_the_plan_cost_when_two_queries_read_one_anchored_group() { + let w = over_m(vec![ + requested( + "sum_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + requested( + "quantile_over_time(0.5, m[5m] @ end())", + 300_000, + Some((60_000, 3_600_000)), + ), + ]); + let models = PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + assert_equal( + "two queries reading one anchored group", + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1, + ); +} + +/// `q + scalar(count(q))` with `q` = `quantile_over_time(0.99, m[5m])`, +/// exact, raw data kept: the `+`'s `Project` reads `q` as its child and +/// the count through `scalar()`. The plan's `Project` keeps `q` as its +/// child (building it handed the count in as the child, as +/// `Operator::map_children` mapped the scalar reference first), and +/// `plan_cost` sizes it by its child, `q`'s 1,000 rows, as the MILP does, +/// not by the count's one row, whatever order the plan lists its edges in. +#[test] +fn the_objective_is_the_plan_cost_on_a_scalar_operand() { + let w = workload(&[( + "quantile_over_time(0.99, m[5m]) + scalar(count(quantile_over_time(0.99, m[5m])))", + 60_000, + "", + )]); + let settings = CostInputs::default(); + let (plan, costs) = selected_plan_objective_and_plan_cost(&w, &settings, true); + assert_equal("scalar operand", costs); + let entries: Vec<_> = w.query_workload.entries().collect(); + let data = w.data_workload.clone().unwrap(); + let mut edges_reversed = plan.clone(); + edges_reversed.edges.reverse(); + let cost_of = |plan| { + asap_cost::plan_cost(plan, &entries, &data, &settings) + .unwrap() + .0 + .cost_per_second + }; + assert_eq!(cost_of(&edges_reversed), cost_of(&plan)); +} diff --git a/crates/planner/tests/exact_only_reads.rs b/crates/planner/tests/exact_only_reads.rs new file mode 100644 index 000000000..a022ea53d --- /dev/null +++ b/crates/planner/tests/exact_only_reads.rs @@ -0,0 +1,112 @@ +//! A query reads the groups Pass 2 lists as exact-only for it +//! (`LogicalASAPMemoCandidates::exact_only_groups_of_query`) only through +//! exact candidates, while another query of the same workload may still +//! read a sketch of those groups: Stage 3's constraint (X), end to end. + +use std::collections::BTreeSet; + +use asap_planner::{plan_installable, FrontendInput, PlanningModels, UserInput}; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::workload::PlanningWorkload; + +/// PromQL `queries`, each with its requirements' JSON (`""`: none), every +/// 60 s at phase 0, over 1,000 gauge series of `m` sampled every second. +fn workload(queries: &[(&str, &str)]) -> PlanningWorkload { + let entries: Vec = queries + .iter() + .map(|(query, requirements)| { + format!( + r#"{{"query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 60000, "evaluation_phase": 0}}}}, + "requirements": {{{requirements}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}"# + ) + }) + .collect(); + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, + "sources": {{"m": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 1000}}}}}}}}"#, + entries.join(",") + )) + .unwrap() +} + +/// The plan of `queries`. +async fn plan(queries: &[(&str, &str)]) -> PhysicalASAPDAG { + let workload = workload(queries); + plan_installable(UserInput::new( + &workload, + FrontendInput::Promql, + PlanningModels::builtin(), + )) + .await + .expect("the workload plans") + .0 + .dag +} + +/// The payloads of the nodes at or below query `query`'s root in `dag`. +fn payloads_read_by(dag: &PhysicalASAPDAG, query: usize) -> Vec { + let mut below = BTreeSet::new(); + let mut pending: Vec = vec![dag.roots[query]]; + while let Some(id) = pending.pop() { + if below.insert(id) { + pending.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == id) + .map(|edge| edge.producer), + ); + } + } + dag.nodes + .iter() + .filter(|node| below.contains(&node.id)) + .map(|node| format!("{:?}", node.payload)) + .collect() +} + +/// The same p90 asked at rank error 0.01 (Q0) and at relative error 0.05 +/// (Q1): KLL's rank error does not meet Q1's relative target, so Pass 2 +/// lists the KLL readout and every group below it as exact-only for Q1. +/// Q1 reads no KLL sketch, even where Q0's plan has one to share. +#[tokio::test] +async fn a_relative_query_reads_no_rank_sketch_of_a_query_sharing_its_groups() { + let dag = plan(&[ + ( + "quantile_over_time(0.9, m[5m])", + r#""accuracy": {"max_error": 0.01, "accuracy_metric": "rank"}"#, + ), + ( + "quantile_over_time(0.9, m[5m])", + r#""accuracy": {"max_error": 0.05, "accuracy_metric": "relative"}"#, + ), + ]) + .await; + let read = payloads_read_by(&dag, 1); + assert!( + !read.iter().any(|payload| payload.contains("Sketch(Kll)")), + "the relative query reads a KLL sketch: {read:?}" + ); +} + +/// An exact p90 (no accuracy target) and the same p90 at rank error 0.01: +/// the exact query reads no sketch, whichever plan the other query gets. +#[tokio::test] +async fn an_exact_query_sharing_groups_with_an_approximate_one_reads_no_sketch() { + let dag = plan(&[ + ("quantile_over_time(0.9, m[5m])", ""), + ( + "quantile_over_time(0.9, m[5m])", + r#""accuracy": {"max_error": 0.01, "accuracy_metric": "rank"}"#, + ), + ]) + .await; + let read = payloads_read_by(&dag, 0); + assert!( + !read.iter().any(|payload| payload.contains("Sketch(")), + "the exact query reads a sketch: {read:?}" + ); +} diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index 9d06c8c46..d17ff5aa4 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -6,6 +6,7 @@ mod time_series; mod columnar_batches; mod e2e_plan; +mod exact_only_reads; mod runtime_install; mod sql_table_scans; mod summary_sharing; diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index ef570d680..c4a2ca803 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -22,6 +22,7 @@ use asap_planner::{ SelectedPlanReport, UserInput, }; use asap_types::ir::operator::Source; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; use futures::{executor::block_on, StreamExt}; @@ -34,11 +35,19 @@ const EVALUATION_MS: i64 = 3_600_000; /// δ 0.01, over 1,000 ingesting `latency_ms` series sampled every second: /// #509 Example 4b when the one query is its p99. fn workload(queries: &[&str]) -> PlanningWorkload { + let every_10_minutes: Vec<(&str, u64)> = + queries.iter().map(|&query| (query, 600_000)).collect(); + workload_every(&every_10_minutes) +} + +/// [`workload`] with each query repeating every `interval_ms` of its own, at +/// phase 0. +fn workload_every(queries: &[(&str, u64)]) -> PlanningWorkload { let entries: Vec = queries .iter() - .map(|query| { + .map(|(query, interval_ms)| { format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": 600000, "evaluation_phase": 0}}}}, + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": {interval_ms}, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, "max_latency_ms": 200.0}}, @@ -248,6 +257,53 @@ async fn the_plan_carries_stage3s_predicted_usage() { assert!(usage.per_node.iter().any(|node| node.node_id == root)); } +/// Example 4b's p99 every 10 minutes and its p50 every 30 minutes: two +/// evaluation contexts. The chain of windows maintained at ingestion time +/// runs once, whoever reads it, and retains the hour both read back +/// ($h_v$, constraint (R)); each context reads it with query-time nodes of +/// its own. +#[tokio::test] +async fn one_chain_serves_two_evaluation_contexts() { + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let (plan, _) = plan_with_report_for( + workload_every(&[ + ("quantile_over_time(0.99, latency_ms[60m])", 600_000), + ("quantile_over_time(0.5, latency_ms[60m])", 1_800_000), + ]), + &capabilities, + ) + .await; + let chains: Vec<_> = plan + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + }) + .collect(); + assert_eq!(chains.len(), 1, "{:#?}", plan.dag.nodes); + assert_eq!(chains[0].retention_ms, Some(3_600_000)); + let ranges_over_the_chain = plan + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + Operator::NonASAP(NonASAPOp::TimeRange { child, .. }) if child == chains[0].id + ) + }) + .count(); + assert_eq!(ranges_over_the_chain, 2); + assert_ne!(plan.dag.roots[0], plan.dag.roots[1]); +} + /// `sum(latency_ms)` reads an instant selector: per series, only the latest /// sample in its 1 s lookback counts. Series `a` has samples at T-900 ms (1) /// and T-100 ms (10), series `b` at T-200 ms (100): the sum is 10 + 100. diff --git a/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs index 1cbb9e09f..bb8265802 100644 --- a/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs +++ b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs @@ -5,7 +5,7 @@ use std::collections::btree_map::Entry; use std::collections::BTreeMap; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::ir::{NonASAPOp, Operator}; use asap_types::stages::{EvaluationGrid, Memo, MemoGroupId}; use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; @@ -92,13 +92,9 @@ pub(crate) fn chain_off_the_evaluation_grid( group: MemoGroupId, input_group: MemoGroupId, ) -> Option<(u64, Option)> { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = memo.groups()[input_group.0 as usize].expression - else { - return None; - }; + let tumbling_window_width_ms = memo.groups()[input_group.0 as usize] + .expression + .chain_tumbling_window_width_ms()?; if !memo.groups()[group.0 as usize] .group_properties .depends_on_evaluation_time diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs index 9784f06af..1e0f9c990 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs @@ -231,14 +231,11 @@ fn chain_path(memo: &Memo, state: MemoGroupId) -> Option { fn existing_chain_widths(memo: &Memo, path: &ChainPath) -> Vec { memo.groups() .iter() - .filter_map(|group| match &group.expression { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if group.expression == path.chain(*tumbling_window_width_ms) => { - Some(*tumbling_window_width_ms) - } - _ => None, + .filter_map(|group| { + group + .expression + .chain_tumbling_window_width_ms() + .filter(|&width_ms| group.expression == path.chain(width_ms)) }) .collect() } diff --git a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs index 5213cae39..f0e48388d 100644 --- a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs @@ -128,11 +128,11 @@ pub fn chain_widths_serving(memo: &Memo, state: MemoGroupId) -> BTreeSet { /// `TimeRange` and `TimeShift` groups, if any. pub fn chain_width_below(memo: &Memo, mut group: MemoGroupId) -> Option { loop { - match &memo.groups()[group.0 as usize].expression { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => return Some(*tumbling_window_width_ms), + let expression = &memo.groups()[group.0 as usize].expression; + if let Some(width_ms) = expression.chain_tumbling_window_width_ms() { + return Some(width_ms); + } + match expression { Operator::NonASAP( NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, ) => group = *child, diff --git a/crates/stage2-physical-optimization/src/group.rs b/crates/stage2-physical-optimization/src/group.rs index 11ca50a94..b99ef844e 100644 --- a/crates/stage2-physical-optimization/src/group.rs +++ b/crates/stage2-physical-optimization/src/group.rs @@ -64,7 +64,7 @@ pub(super) fn ingestion_time_offered_per_group( /// `AcrossEvaluations`. It does for a chain, for a group evaluated on a /// known grid (`evaluation_grid_of_group`) and for a time-independent /// group (its stored result is read wherever it is final, -/// `PhysicalCandidate::delivers_on`), unless +/// `PhysicalASAPMemoCandidates::delivers_on`), unless /// the group is read under an absolute `@ T` /// ([`read_at_absolute_time_per_group`]). pub(super) fn across_evaluations_offered_per_group( diff --git a/crates/stage2-physical-optimization/src/lib.rs b/crates/stage2-physical-optimization/src/lib.rs index b3cc32315..66e05aa4a 100644 --- a/crates/stage2-physical-optimization/src/lib.rs +++ b/crates/stage2-physical-optimization/src/lib.rs @@ -44,7 +44,7 @@ use candidate::{offered_physical_properties, OfferedPhysicalProperties}; /// outside a dimension's declared value set (`AcrossEvaluations` on a /// candidate of a time-dependent group with no known grid) is not offered /// and not recorded. Costs nothing, and cannot fail; alignment with each -/// execution context is Stage 3's check (`PhysicalCandidate::delivers_on`). +/// execution context is Stage 3's check (`PhysicalASAPMemoCandidates::delivers_on`). pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, data: &DataWorkload, diff --git a/crates/stage2-physical-optimization/src/tests/candidates.rs b/crates/stage2-physical-optimization/src/tests/candidates.rs index e98073b93..02ec2887d 100644 --- a/crates/stage2-physical-optimization/src/tests/candidates.rs +++ b/crates/stage2-physical-optimization/src/tests/candidates.rs @@ -31,7 +31,7 @@ use crate::test_fixtures::{ /// materialized across evaluations. The scan and the chain are /// time-independent, so ingestion time is offered too, and so is /// `AcrossEvaluations` without a grid: kept up to date at ingestion time, -/// their result is final at any time (`PhysicalCandidate::delivers_on`). A +/// their result is final at any time (`PhysicalASAPMemoCandidates::delivers_on`). A /// summary build is not offered (IT, None): fed at ingestion time /// it keeps its state, so the chain is offered (IT, AcrossEvaluations) /// only; the scan's (IT, None) feeds the chain built at ingestion time. diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 40f0b350c..2af18c0d0 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -1,120 +1,283 @@ -//! The baseline objective's coefficients +//! The objective's coefficients //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 3, "Stage -//! 3: baseline selection and plan output"): $w_v$ of every physical -//! candidate and $c_{u,c,i}$ of every edge a plan may select, from -//! `asap-cost` under cost model 1. +//! sections 3 and 5, "Cost and retention") under cost model 1, from +//! `asap-cost`: per execution of a candidate in a context +//! ([`crate::selectable`]), per edge in a context, and per amount a +//! candidate retains: //! -//! Each candidate is costed as one plan node ([`candidate_node`], -//! [`asap_cost::node_resource_usage_and_cost`]) over the logical output of each input: the -//! most any costed candidate it may read there (`Memo::producers`) -//! outputs (rows, bytes and, for a state, its family). That is $w_v$, which -//! does not depend on the producers chosen. The part of a consumer's cost -//! that changes with the producer read is on the edge: $c_{u,c,i}$ is the -//! consumer's cost with input $i$ read from $u$'s own output, minus $w_c$. -//! It is exact when a consumer's cost changes with one input's producer -//! only (every consumer of a state reads one state input). Every state -//! candidate has its own θ (a merge, a `TimeRange` over a chain carry the -//! θ of the states they read, and read only producers with it), so a -//! consumer above a sketch build is costed at that build's θ. +//! $$\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} +//! \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum_{\kappa,u,c,i} +//! \rho_\kappa c_{\kappa,u,c,i} y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s +//! r_s,$$ //! -//! How often a candidate runs is the baseline's fixed rate: at ingestion -//! time, one second of arriving rows per second; at query time, once per -//! evaluation of the queries reaching its group (once per request when it -//! is materialized across evaluations). A materialized candidate and a -//! query-time `Scan` retain the most their selectable consumers read back -//! ([`retention_need_ms`]). Execution contexts make both per context. +//! less the request overhead every plan pays (`crate::select_plan` adds +//! it). Each candidate is costed as +//! one plan node ([`candidate_node`], +//! [`asap_cost::node_resource_usage_and_cost`]) over a baseline of each +//! input: the most any producer it may read (on its admissible edges) +//! outputs (rows, bytes and, for a state, its θ), at its execution's rate +//! $\rho_\kappa$ ([`executions_per_second`]). That does not depend on the +//! producers chosen. An ingestion-time consumer reads one batch of raw +//! data, also from a `Scan` materialized at ingestion time +//! ([`reads_one_batch_of`]). The part of a consumer's cost that changes +//! with the producer read is on the edge: $\rho_\kappa c_{\kappa,u,c,i}$ is +//! the consumer's cost in $\kappa$ with input $i$ read from $u$'s own +//! output, minus its own. It is exact when a consumer's cost changes with +//! one input's producer only (every consumer of a state reads one state +//! input). Every state candidate has its own θ (a merge, a `TimeRange` over +//! a chain carry the θ of the states they read, and may read only +//! producers with it), so the baseline over its admissible producers is +//! that θ's. +//! +//! A query-time `Scan` that is not materialized has one consumer per +//! context, and reads in each run what that consumer reads back in one +//! run ([`edge_scan_read_ms`]): that read is on the edge to the consumer, +//! a scan's read cost on its edges. A materialized one (`OneEvaluation`, +//! shared within a context) reads the most any consumer it may have in +//! the context reads, a constant of the execution: exact when that +//! consumer is selected. +//! +//! A candidate that retains time-indexed rows +//! (`PhysicalASAPDAGNode::retains_rows`) is costed retaining the most its +//! consumers read back ([`edge_retention_need_ms`]), the largest $h_v$ (R) +//! allows. Retention is linear in that amount: $\pi_v$ is what retaining +//! one more output (a window, or a second) costs there, and $W^{\mathrm{mat}}_v$ of a +//! materialized candidate is its cost less $\pi_v h_v$. A query-time +//! scan's retention is raw data on disk, kept once per source for the +//! selected scan that keeps the most: $r_s \ge \pi_v h_v$ +//! ([`asap_cost::raw_data_disk_cost_per_second`]). A candidate one of whose +//! consumers reads it unbounded (a `Scan` read whole, a `TimeShift` +//! anchored with `@`) has no $h_v$: it is costed retaining everything. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; -use asap_cost::plan::execution_demand::executions_per_second; +use asap_cost::plan::execution_demand::{ + context_executions_per_second, executions_per_second as rate_of_materialized, +}; use asap_cost::{ - maintained_window_ms, node_resource_usage_and_cost, raw_data_disk_cost_per_second, CostInputs, + node_resource_usage_and_cost, output_interval_ms, raw_data_disk_cost_per_second, CostInputs, NodeResourceUsageAndCost, ProducerOutput, }; use asap_types::ir::operator::Source; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::plan::materialized_view::RefreshWindow; use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, }; use asap_types::stages::{ - Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, + queries_of_context, ExecutionContext, Lifetime, Memo, MemoGroupId, PhysicalASAPMemoCandidates, + PhysicalCandidateId, PhysicalEdgeId, }; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::plan::candidate_node; -use crate::retention::{retention_need_ms, scan_read_ms}; +use crate::retention::{ + edge_retention_need_ms, edge_scan_read_ms, least_retention_ms_per_execution, ms_to_seconds, +}; +use crate::selectable::{Execution, SelectableExecutions}; -/// $w_v$ and $c_{u,c,i}$. -pub(crate) struct SelectionCosts { - /// $w_v$, cost per second, by [`PhysicalCandidateId`], or why - /// `asap-cost` could not cost it (the MILP leaves such a candidate out). - pub(crate) candidate_cost_per_second: Vec>, - /// $c_{u,c,i}$, the change of its consumer's cost per second, of each - /// selectable edge whose producer and consumer are costable; an edge - /// missing here cannot be selected. - pub(crate) edge_cost_change_per_second: HashMap, - /// By [`PhysicalCandidateId`], for a query-time `Scan`: its source and - /// the cost per second of keeping the raw data it reads on disk - /// ([`asap_cost::raw_data_disk_cost_per_second`]). A plan keeps each - /// source once, for the selected scan that keeps the most, as - /// `plan_cost` charges it. - pub(crate) raw_data_on_disk: Vec>, - /// The runtime's fixed overhead per query request, per second - /// ([`asap_cost::request_overhead_cost_per_second`]): every plan pays - /// it. - pub(crate) request_overhead_cost_per_second: f64, +/// The objective's coefficients +/// (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +/// section 5, "Cost and retention"), per second. +pub(crate) struct ExecutionCosts { + /// $\rho_\kappa c_{\kappa,v}$ of each execution, $W^{\mathrm{mat}}_v$ + /// of an execution in `Materialized(v)`, by position in + /// `SelectableExecutions::executions`; or why `asap-cost` could not + /// cost it (the MILP leaves it out). + pub(crate) per_execution: Vec>, + /// $\rho_\kappa c_{\kappa,u,c,i}$ of each edge, by position in + /// `SelectableExecutions::edges`; `None` when it could not be costed + /// (the MILP leaves it out). + pub(crate) per_edge: Vec>, + /// $\pi_v$, the cost per second of each second retained, of each + /// candidate that retains rows (`PhysicalASAPDAGNode::retains_rows`) + /// only within a bound: it has $h_v$ and constraint (R). One whose + /// consumer reads it unbounded is costed retaining everything. + pub(crate) retention_cost_per_retained_second: HashMap, + /// $\operatorname{need}_{\kappa,u,c,i}$ in ms of each edge, by position + /// in `SelectableExecutions::edges` ([`edge_retention_need_ms`]); + /// `None` when its producer retains no rows or the edge reads it + /// unbounded. + pub(crate) retention_need_ms_per_edge: Vec>, + /// Per candidate that retains rows, its output interval Δ in ms + /// (`asap_cost::output_interval_ms`; a chain's window, the window an + /// ingestion-time view maintains, else a second): a store always keeps a + /// stored result's newest output whatever its retention, so a selected + /// one retains at least Δ, as `plan_cost` charges at least one retained + /// output ([`Self::retention_ms_for_need`]). Example: an ingestion-time + /// chain of 60 s windows read only at its newest window retains 60 s, + /// not 0. + pub(crate) newest_output_retention_ms: HashMap, + /// The least retention, in ms, over each execution's costed reads + /// ([`Self::retention_ms_for_need`] of each), by position in + /// `SelectableExecutions::executions` ([`least_retention_ms_per_execution`]). + pub(crate) least_retention_ms_per_execution: Vec>, + /// The source of each query-time `Scan`: its retention is raw data on + /// disk, which a plan keeps once per source for the selected scan that + /// keeps the most, as `plan_cost` charges it ($r_s$). + pub(crate) raw_data_source: HashMap, + /// The cost per second of the raw data a query-time `Scan` without + /// $h_v$ keeps on disk, retaining everything. + pub(crate) unbounded_raw_data_on_disk_cost_per_second: HashMap, } -/// [`SelectionCosts`] over `selectable` (each admissible edge a plan may -/// select, by consumer and input). Errs on nothing: a candidate -/// `asap-cost` cannot cost is reported in -/// [`SelectionCosts::candidate_cost_per_second`]. -pub(crate) fn selection_costs( +impl ExecutionCosts { + /// What candidate `v` retains, in ms, for a read needing `need_ms`: at + /// least its newest output ([`Self::newest_output_retention_ms`]), + /// which a store always keeps. + /// + /// Example: a chain of 60 s windows read at its newest window (need 0) + /// retains 60,000 ms; read over the last 5 minutes, 300,000 ms. + pub(crate) fn retention_ms_for_need(&self, v: PhysicalCandidateId, need_ms: u64) -> u64 { + self.newest_output_retention_ms + .get(&v) + .map_or(need_ms, |&newest_ms| need_ms.max(newest_ms)) + } + + /// $\pi_v h_v$ of candidate `v` retaining `retention_ms`, per second; 0 + /// for a candidate with no retention cost. + pub(crate) fn retention_cost_per_second( + &self, + v: PhysicalCandidateId, + retention_ms: u64, + ) -> f64 { + self.retention_cost_per_retained_second + .get(&v) + .map_or(0.0, |per_retained_second| { + per_retained_second * ms_to_seconds(retention_ms) + }) + } +} + +/// The objective's coefficients of every execution and edge of +/// `selectable` ([`ExecutionCosts`]; module doc). Errs on nothing: what +/// `asap-cost` cannot cost is reported in [`ExecutionCosts::per_execution`] +/// or left out of [`ExecutionCosts::per_edge`]. +pub(crate) fn execution_costs( physical: &PhysicalASAPMemoCandidates, - selectable: &[PhysicalEdgeId], + selectable: &SelectableExecutions, entries: &[QueryWorkloadEntry], data: &DataWorkload, cost_inputs: &CostInputs, - request_overhead_cost_per_second: f64, -) -> SelectionCosts { +) -> ExecutionCosts { let memo = &physical.logical_candidates.memo; let count = physical.physical_candidates.len(); - let mut out_edges: Vec> = vec![Vec::new(); count]; - for &edge in selectable { - out_edges[physical.edge(edge).producer_candidate_id.0].push(edge); - } - // The candidates that may read each candidate. - let readers_of = |v: PhysicalCandidateId| { - out_edges[v.0] - .iter() - .map(|&edge| physical.edge(edge).consumer_candidate_id) - }; + let executions = selectable.executions(); // Each candidate's node, its inputs numbered 0.. (the ids do not - // matter to `node_resource_usage_and_cost`), with the retention it would need. - let nodes: Vec = (0..count) + // matter to `node_resource_usage_and_cost`), retaining nothing yet. + let node_retaining = |v: PhysicalCandidateId, retention_ms: Option| { + let inputs: Vec = (0..physical.input_count(v)).collect(); + let mut node = candidate_node(physical, v, v.0, &inputs); + node.retention_ms = retention_ms; + node + }; + let unretained_nodes: Vec = (0..count) + .map(|v| node_retaining(PhysicalCandidateId(v), None)) + .collect(); + let retains_rows: Vec = unretained_nodes + .iter() + .map(PhysicalASAPDAGNode::retains_rows) + .collect(); + let query_time_scan: Vec = (0..count) + .map(|v| physical.is_query_time_scan(PhysicalCandidateId(v))) + .collect(); + let producer_of = |position: usize| { + let edge = selectable.edges()[position]; + (edge.edge, executions[edge.producer].candidate) + }; + // Each edge's need, computed once per edge of the physical memo: an + // edge is read in every context it is selectable in. + let mut need_ms_of_edge: HashMap> = HashMap::new(); + let retention_need_ms_per_edge: Vec> = (0..selectable.edges().len()) + .map(|position| { + let (edge, u) = producer_of(position); + if !retains_rows[u.0] { + return None; + } + *need_ms_of_edge + .entry(edge) + .or_insert_with(|| edge_retention_need_ms(physical, entries, edge)) + }) + .collect(); + // Δ of each candidate that retains rows, the newest output a store + // always keeps (`ExecutionCosts::newest_output_retention_ms`). + let newest_output_retention_ms: HashMap = (0..count) + .filter(|&v| retains_rows[v]) .map(PhysicalCandidateId) .map(|v| { - let inputs: Vec = (0..physical.input_count(v)).collect(); - let mut node = candidate_node(physical, v, v.0, &inputs); - if node.retains_rows() { - node.retention_ms = retention_need_ms(physical, entries, v, readers_of); - } - node + let node = &unretained_nodes[v.0]; + let window_ms = match node.output_state.timing { + ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), + ExecutionTiming::QueryTime => None, + }; + (v, output_interval_ms(&node.payload, window_ms)) }) .collect(); - // Each query-time `Scan`'s read per execution: the most any selectable - // consumer reads (no range-evaluation span), charged once. - let scan_read_ms_of: Vec> = nodes + // The most any consumer of each candidate that retains rows reads back + // ($h_v$ at its largest), at least its newest output: `None` when one + // reads it unbounded. + let mut most_read_ms: HashMap> = newest_output_retention_ms .iter() + .map(|(&v, &newest_ms)| (v, Some(newest_ms))) + .collect(); + for (index, _, need_ms) in selectable.reads_back(&retention_need_ms_per_edge, entries) { + if let Some(most) = most_read_ms.get_mut(&executions[index].candidate) { + *most = most.zip(need_ms).map(|(most, need)| most.max(need)); + } + } + // How far back a query-time `Scan` reads for the consumer of each edge + // out of it, in one run (`None`: read whole), once per memo edge. + let mut scan_read_ms_of_edge: HashMap> = HashMap::new(); + let scan_read_ms_per_edge: Vec>> = (0..selectable.edges().len()) + .map(|position| { + let (edge, u) = producer_of(position); + query_time_scan[u.0].then(|| { + *scan_read_ms_of_edge + .entry(edge) + .or_insert_with(|| edge_scan_read_ms(physical, edge)) + }) + }) + .collect(); + // Each query-time `Scan`'s read for its baseline output (what its + // consumers' baselines read): the most any of its consumers reads. + let mut baseline_scan_read_ms: HashMap> = (0..count) + .filter(|&v| query_time_scan[v]) + .map(|v| (PhysicalCandidateId(v), Some(0))) + .collect(); + // A query-time `Scan`'s read per execution: one that is not + // materialized has one reader per context, and its read is on the edge + // to it (0 here); one materialized, shared by its readers, reads the + // most any of them reads in one run. + let mut execution_read_ms: Vec> = executions + .iter() + .map(|execution| query_time_scan[execution.candidate.0].then_some(0)) + .collect(); + for (position, read_ms) in scan_read_ms_per_edge.iter().enumerate() { + let Some(read_ms) = *read_ms else { + continue; + }; + let producer = selectable.edges()[position].producer; + let u = executions[producer].candidate; + let most = baseline_scan_read_ms + .get_mut(&u) + .expect("a query-time scan"); + *most = most.zip(read_ms).map(|(most, read)| most.max(read)); + if physical.physical_candidates[u.0] + .physical_properties + .is_materialized() + { + let of_execution = &mut execution_read_ms[producer]; + *of_execution = of_execution.zip(read_ms).map(|(most, read)| most.max(read)); + } + } + let baseline_scan_read_of = + |v: PhysicalCandidateId| baseline_scan_read_ms.get(&v).copied().flatten(); + let nodes: Vec = unretained_nodes + .into_iter() .enumerate() - .map(|(v, node)| { - let query_time_scan = node.output_state.timing == ExecutionTiming::QueryTime - && matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. })); - query_time_scan - .then(|| scan_read_ms(physical, PhysicalCandidateId(v), readers_of)) - .flatten() + .map(|(v, mut node)| { + node.retention_ms = most_read_ms.get(&PhysicalCandidateId(v)).copied().flatten(); + node }) .collect(); // Each group's expression as a payload, for its logical output. @@ -132,123 +295,348 @@ pub(crate) fn selection_costs( let order = memo .topological_order() .unwrap_or_else(|| (0..memo.groups().len() as u32).map(MemoGroupId).collect()); + // A candidate's (own, raw data on disk) cost per second as `node` over + // `inputs`, a query-time `Scan` reading `scan_read_ms` per execution. + let cost_reading = |v: PhysicalCandidateId, + node: &PhysicalASAPDAGNode, + inputs: &[ProducerOutput<'_>], + per_second: f64, + scan_read_ms: Option| { + cost_node( + physical, + v, + node, + inputs, + per_second, + scan_read_ms, + data, + cost_inputs, + ) + .and_then(|usage| cost_per_second(&usage, cost_inputs)) + }; + let cost = |v: PhysicalCandidateId, + node: &PhysicalASAPDAGNode, + inputs: &[ProducerOutput<'_>], + per_second: f64| { + cost_reading(v, node, inputs, per_second, baseline_scan_read_of(v)) + }; + // Each candidate's usage of one computation (what it outputs does not + // depend on how often it runs) over its baseline inputs. let mut usage: Vec, String>>> = (0..count).map(|_| None).collect(); - // Each candidate's logical inputs, as costed for $w_v$; each edge into - // it replaces one. - let mut inputs_of: Vec>>> = (0..count).map(|_| None).collect(); + let mut baseline: Vec>>> = vec![None; count]; for group in order { for &v in &candidates_of_group[group.0 as usize] { - let inputs = logical_inputs(physical, &group_payloads, &candidates_of_group, &usage, v); - usage[v.0] = Some(match &inputs { + baseline[v.0] = baseline_inputs(memo, physical, &group_payloads, &nodes, &usage, v); + usage[v.0] = Some(match &baseline[v.0] { Some(inputs) => cost_node( physical, + v, &nodes[v.0], inputs, - entries, + 1.0, + baseline_scan_read_of(v), data, cost_inputs, - scan_read_ms_of[v.0], ), - None => Err("an input group has no costable candidate".to_string()), + None => Err("an input has no costable producer".to_string()), }); - inputs_of[v.0] = inputs; } } - let candidate_cost_per_second: Vec> = usage + // Each execution's (own, raw data on disk) cost per second at its + // rate, over its baseline inputs, and that rate. + let at_rate: Vec> = executions .iter() - .map(|usage| match usage { - Some(Ok(usage)) => cost_per_second(usage), - Some(Err(error)) => Err(error.clone()), - None => Err("it reads a group on a cycle of candidates".to_string()), + .zip(&execution_read_ms) + .map(|(&execution, &read_ms)| { + let v = execution.candidate; + if let Some(Err(error)) = &usage[v.0] { + return Err(error.clone()); + } + let per_second = executions_per_second( + physical, + entries, + selectable, + execution, + cost_inputs.non_repeating_query_horizon_seconds, + )?; + let inputs = baseline[v.0] + .as_ref() + .ok_or("it reads a group on a cycle of candidates")?; + let (own, raw_data) = cost_reading(v, &nodes[v.0], inputs, per_second, read_ms)?; + Ok((own, raw_data, per_second)) }) .collect(); - let raw_data_on_disk = usage + // The one execution of each candidate materialized across evaluations, + // by position. + let own_execution: HashMap = executions .iter() - .map(|usage| { - let (source, bytes) = usage.as_ref()?.as_ref().ok()?.raw_scan?; - let cost = raw_data_disk_cost_per_second(bytes, cost_inputs)?; - Some((source.clone(), cost)) + .enumerate() + .filter(|(_, execution)| { + physical.physical_candidates[execution.candidate.0] + .physical_properties + .is_across_evaluations() }) + .map(|(index, execution)| (execution.candidate, index)) .collect(); - let mut edge_cost_change_per_second = HashMap::new(); - for &id in selectable { - let e = physical.edge(id); - let (u, c) = (e.producer_candidate_id, e.consumer_candidate_id); - let (Some(Ok(produced)), Ok(own)) = (&usage[u.0], &candidate_cost_per_second[c.0]) else { + // π_v of a candidate retaining rows every consumer reads within a bound: + // what one more Δ ms of retention (one more output; a retention is a + // whole number of outputs) costs, per second retained, at its + // retention's largest; its whole own cost when it is materialized, its + // raw data on disk for a query-time scan (each of its runs reads what + // it reads at that retention, ρ_κ c_{κ,v}). + let mut retention_cost_per_retained_second = HashMap::new(); + for (&v, &most_read_ms) in &most_read_ms { + let Some(most_read_ms) = most_read_ms else { continue; }; - let Some(mut inputs) = inputs_of[c.0].clone() else { - continue; + let delta_ms = newest_output_retention_ms[&v]; + let longer = node_retaining(v, Some(most_read_ms + delta_ms)); + let delta_seconds = ms_to_seconds(delta_ms); + let per_retained_second = match physical.physical_candidates[v.0] + .physical_properties + .is_across_evaluations() + { + true => own_execution.get(&v).and_then(|&index| { + let (own, raw_data, per_second) = at_rate[index].as_ref().ok()?; + let inputs = baseline[v.0].as_ref()?; + let (longer_own, longer_raw_data) = cost(v, &longer, inputs, *per_second).ok()?; + Some((longer_own + longer_raw_data - own - raw_data) / delta_seconds) + }), + false => usage[v.0] + .as_ref() + .and_then(|usage| usage.as_ref().ok()) + .and_then(|_| { + let inputs = baseline[v.0].as_ref()?; + let (_, raw_data) = cost(v, &nodes[v.0], inputs, 1.0).ok()?; + let (_, longer_raw_data) = cost(v, &longer, inputs, 1.0).ok()?; + Some((longer_raw_data - raw_data) / delta_seconds) + }), }; - inputs[e.input_index_of_consumer] = - produced.producer_output(&nodes[u.0].payload, &nodes[u.0].output_schema); - let with_u = cost_node( - physical, - &nodes[c.0], - &inputs, - entries, - data, - cost_inputs, - scan_read_ms_of[c.0], - ) - .and_then(|usage| cost_per_second(&usage)); - if let Ok(with_u) = with_u { - edge_cost_change_per_second.insert(id, with_u - own); + if let Some(per_retained_second) = per_retained_second { + retention_cost_per_retained_second.insert(v, per_retained_second.max(0.0)); } } - SelectionCosts { - candidate_cost_per_second, - edge_cost_change_per_second, - raw_data_on_disk, - request_overhead_cost_per_second, + // Each query-time scan's source and, without $h_v$, the raw data it + // keeps on disk retaining everything. + let mut raw_data_source = HashMap::new(); + let mut unbounded_raw_data_on_disk_cost_per_second = HashMap::new(); + for v in (0..count) + .filter(|&v| query_time_scan[v]) + .map(PhysicalCandidateId) + { + let Some(Ok(scan)) = &usage[v.0] else { + continue; + }; + let Some((source, _)) = scan.raw_scan else { + continue; + }; + raw_data_source.insert(v, source.clone()); + if !retention_cost_per_retained_second.contains_key(&v) { + if let Some(Ok((_, raw_data))) = baseline[v.0] + .as_ref() + .map(|inputs| cost(v, &nodes[v.0], inputs, 1.0)) + { + unbounded_raw_data_on_disk_cost_per_second.insert(v, raw_data); + } + } } + let per_execution: Vec> = executions + .iter() + .zip(&at_rate) + .map(|(execution, at_rate)| { + let &(own, raw_data, _) = at_rate.as_ref().map_err(Clone::clone)?; + let v = execution.candidate; + let per_retained_second = retention_cost_per_retained_second.get(&v); + Ok(if query_time_scan[v.0] { + // A query-time scan's raw data is charged per source. + own + } else if let (Some(per_retained_second), true) = ( + per_retained_second, + physical.physical_candidates[v.0] + .physical_properties + .is_across_evaluations(), + ) { + let retention_ms = + most_read_ms[&v].expect("a candidate with π_v is read within a bound"); + own + raw_data - per_retained_second * ms_to_seconds(retention_ms) + } else { + own + raw_data + }) + }) + .collect(); + let per_edge: Vec> = selectable + .edges() + .iter() + .zip(&scan_read_ms_per_edge) + .map(|(edge, scan_read_ms)| { + let (consumer, producer) = (edge.consumer, edge.producer); + let e = physical.edge(edge.edge); + let (u, c) = (e.producer_candidate_id, e.consumer_candidate_id); + per_execution[producer].as_ref().ok()?; + let &(own, raw_data, per_second) = at_rate[consumer].as_ref().ok()?; + let produced = usage[u.0].as_ref()?.as_ref().ok()?; + let mut inputs = baseline[c.0].clone()?; + if !reads_one_batch_of(physical, &nodes, c, u) { + inputs[e.input_index_of_consumer] = + produced.producer_output(&nodes[u.0].payload, &nodes[u.0].output_schema); + } + let (with_u_own, with_u_raw_data) = cost(c, &nodes[c.0], &inputs, per_second).ok()?; + // A scan that is not materialized reads, in its one consumer's + // run, what that consumer reads: on this edge. + let scan_read = match scan_read_ms { + Some(read_ms) if nodes[u.0].lifetime == Lifetime::None => { + let &(scan_own, _, scan_per_second) = at_rate[producer].as_ref().ok()?; + let (reading_own, _) = cost_reading( + u, + &nodes[u.0], + baseline[u.0].as_ref()?, + scan_per_second, + *read_ms, + ) + .ok()?; + reading_own - scan_own + } + _ => 0.0, + }; + Some(with_u_own + with_u_raw_data - own - raw_data + scan_read) + }) + .collect(); + let mut costs = ExecutionCosts { + per_execution, + per_edge, + retention_cost_per_retained_second, + retention_need_ms_per_edge, + newest_output_retention_ms, + least_retention_ms_per_execution: Vec::new(), + raw_data_source, + unbounded_raw_data_on_disk_cost_per_second, + }; + costs.least_retention_ms_per_execution = + least_retention_ms_per_execution(selectable, &costs, entries); + costs +} + +/// $\rho$ of `execution`'s context, how often it runs, from the queries it +/// serves ([`context_executions_per_second`], which `plan_cost` uses too): +/// in an evaluation context, for the queries on its grid whose walk over +/// the memo reaches the execution's group in their evaluation context +/// (`SelectableExecutions::readers_of`, [`asap_types::stages::query_reads`], +/// the walk `plan_cost` takes over the plan), at their evaluations; for +/// the inputs of a `TimeShift` anchored with `@`, once per request of the +/// queries on the shift's grid that reach the shift. The memo walk goes +/// through every candidate of a group, so a query reading a group through +/// one candidate counts for its other candidates too; +/// in `Materialized(m)`, as often as $m$ runs +/// ([`asap_cost::plan::execution_demand::executions_per_second`]); in the +/// ingestion pipeline, once per second of arriving rows. +/// +/// Example: queries every 10 s, 10 s and 30 s reaching a scan: 1/10 per +/// second in the 10 s context, 1/30 in the 30 s one. +pub(crate) fn executions_per_second( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + selectable: &SelectableExecutions, + execution: Execution, + non_repeating_query_horizon_seconds: f64, +) -> Result { + let memo = &physical.logical_candidates.memo; + let group_of = + |v: PhysicalCandidateId| physical.physical_candidates[v.0].logical_candidate_id.group; + let reaching = |v: PhysicalCandidateId| { + memo.groups()[group_of(v).0 as usize] + .reached_by_queries() + .iter() + .copied() + }; + let entries_of = |queries: Vec| -> Vec<&QueryWorkloadEntry> { + queries + .into_iter() + .filter_map(|query| entries.get(query)) + .collect() + }; + let rate = match selectable.context(execution.context) { + ExecutionContext::Evaluation { grid } => match selectable.request_anchor(execution.context) + { + Some((at, queries)) => context_executions_per_second( + Some(at), + entries_of(queries.iter().map(|query| query.0).collect()), + non_repeating_query_horizon_seconds, + ), + None => context_executions_per_second( + None, + entries_of(queries_of_context( + grid.period_ms, + selectable.readers_of(group_of(execution.candidate), |anchor| anchor.is_none()), + entries, + )), + non_repeating_query_horizon_seconds, + ), + }, + ExecutionContext::Materialized(m) => { + let properties = physical.physical_candidates[m.0].physical_properties; + rate_of_materialized( + properties.timing(), + properties.lifetime(), + entries_of(reaching(m).collect()), + non_repeating_query_horizon_seconds, + ) + } + // Each arriving row once. + ExecutionContext::IngestionPipeline(_) => Ok(1.0), + }; + rate.map_err(|error| error.to_string()) } -/// The logical output of each input of `v` over the costed candidates -/// (`usage`) it may read there (`Memo::producers`); `None` when one has -/// none. -fn logical_inputs<'a>( - physical: &'a PhysicalASAPMemoCandidates, +/// Candidate `v`'s baseline inputs, by input index: over the producers it +/// may read (its admissible edges) whose usage is known, the most any +/// outputs ([`logical_output`]); `None` when an input has none. A producer +/// of which `v` reads one batch ([`reads_one_batch_of`]) is left out. +fn baseline_inputs<'a>( + memo: &'a Memo, + physical: &PhysicalASAPMemoCandidates, group_payloads: &'a [PhysicalASAPOperatorPayload], - candidates_of_group: &[Vec], + nodes: &[PhysicalASAPDAGNode], usage: &[Option, String>>], v: PhysicalCandidateId, ) -> Option>> { - let memo = &physical.logical_candidates.memo; - let consumer = physical.physical_candidates[v.0].logical_candidate_id; - input_groups(memo, physical, v) + physical + .operator(v) + .children() .into_iter() .enumerate() - .map(|(input, group)| { - let readable: HashSet<_> = memo.producers(consumer, input).into_iter().collect(); + .map(|(i, input)| { logical_output( - &group_payloads[group.0 as usize], - &memo.groups()[group.0 as usize].group_properties.schema, - candidates_of_group[group.0 as usize] + &group_payloads[input.0 as usize], + &memo.groups()[input.0 as usize].group_properties.schema, + physical + .admissible_edges_into(v, i) .iter() - .filter(|u| { - readable.contains(&physical.physical_candidates[u.0].logical_candidate_id) - }) + .map(|&edge| physical.edge(edge).producer_candidate_id) + .filter(|&u| !reads_one_batch_of(physical, nodes, v, u)) .filter_map(|u| usage[u.0].as_ref()?.as_ref().ok()), ) }) .collect() } -/// The input groups of candidate `v`, by input index. -fn input_groups( - memo: &Memo, +/// Whether consumer `c` reads only one batch of producer `u`'s output +/// ([`asap_cost::reads_one_batch_of`], over `u`'s input group's +/// operator): such an edge costs what reading a `Scan` that is not +/// materialized does. +fn reads_one_batch_of( physical: &PhysicalASAPMemoCandidates, - v: PhysicalCandidateId, -) -> Vec { - memo.candidate(physical.physical_candidates[v.0].logical_candidate_id) - .expression - .operator + nodes: &[PhysicalASAPDAGNode], + c: PhysicalCandidateId, + u: PhysicalCandidateId, +) -> bool { + let memo = &physical.logical_candidates.memo; + let input = physical + .operator(u) .children() - .into_iter() - .copied() - .collect() + .first() + .map(|group| &memo.groups()[group.0 as usize].expression); + asap_cost::reads_one_batch_of(&nodes[c.0], &nodes[u.0], input) } /// An input's logical output over its readable costed candidates' `usages`: the most @@ -280,69 +668,67 @@ fn logical_output<'a: 'b, 'b>( output } -/// The usage of the candidate `node` over `inputs` (module doc): run -/// [`executions_per_second`] for the queries reaching its group, over the -/// window an ingestion-time grouping maintains -/// ([`asap_cost::maintained_window_ms`], over the memo's groups) or, for a +/// Candidate `v`'s usage as `node` over `inputs`, computed `per_second` +/// times per second, over its refresh window at ingestion time +/// ([`ingestion_window_ms`]) or, for a /// query-time `Scan`, `scan_read_ms` per execution. +#[allow(clippy::too_many_arguments)] fn cost_node<'a>( physical: &PhysicalASAPMemoCandidates, + v: PhysicalCandidateId, node: &'a PhysicalASAPDAGNode, inputs: &[ProducerOutput<'a>], - entries: &[QueryWorkloadEntry], + per_second: f64, + scan_read_ms: Option, data: &DataWorkload, cost_inputs: &'a CostInputs, - scan_read_ms: Option, ) -> Result, String> { - let memo = &physical.logical_candidates.memo; - let candidate = node - .selected_physical_candidate - .expect("a candidate's node names it"); - let group = physical.physical_candidates[candidate.0] - .logical_candidate_id - .group; - let per_second = executions_per_second( - node.output_state.timing, - node.lifetime, - memo.groups()[group.0 as usize] - .reached_by_queries() - .iter() - .filter_map(|&query| entries.get(query)), - cost_inputs.non_repeating_query_horizon_seconds, - ) - .map_err(|error| error.to_string())?; let window_ms = match node.output_state.timing { ExecutionTiming::QueryTime => scan_read_ms, - // A chain's group computes the chain (Pass 2 adds chain candidates - // only to it), so the group's window is the chain's. - ExecutionTiming::IngestionTime => maintained_window_ms( - group, - |group: MemoGroupId| &memo.groups()[group.0 as usize].expression, - |group| { - memo.groups()[group.0 as usize] - .expression - .children() - .first() - .map(|g| **g) - }, - ), + ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), }; node_resource_usage_and_cost(node, inputs, (1, per_second), window_ms, data, cost_inputs) .map_err(|error| error.to_string()) } -/// $w_v$ from a candidate's usage: its cost per second under cost model 1 -/// (the raw data a query-time scan keeps on disk is charged per source, -/// [`SelectionCosts::raw_data_on_disk`]). -fn cost_per_second(usage: &NodeResourceUsageAndCost<'_>) -> Result { - let cost = usage +/// The time between two outputs of ingestion-time candidate `v`: the +/// width of its refresh window, by the rule the installed plan refreshes +/// it on (`RefreshWindow::computed_from` over the candidates, so +/// `plan_cost` charges the same); `None` without one (one output per +/// second). +fn ingestion_window_ms( + physical: &PhysicalASAPMemoCandidates, + v: PhysicalCandidateId, +) -> Option { + RefreshWindow::computed_from(physical, v).map(|window| window.width_ms) +} + +/// A candidate's cost per second from its usage under cost model 1: its +/// own, and the raw data a query-time scan keeps on disk +/// ([`asap_cost::raw_data_disk_cost_per_second`]). +fn cost_per_second( + usage: &NodeResourceUsageAndCost<'_>, + cost_inputs: &CostInputs, +) -> Result<(f64, f64), String> { + let own = usage .cost .cost_per_second .ok_or("the selection MILP costs under cost model 1 only")?; - match cost.is_finite() && cost >= 0.0 { - true => Ok(cost), + let raw_data = match usage.raw_scan { + Some((_, bytes)) => raw_data_disk_cost_per_second(bytes, cost_inputs) + .ok_or("the selection MILP costs under cost model 1 only")?, + None => 0.0, + }; + match [own, raw_data] + .iter() + .all(|cost| cost.is_finite() && *cost >= 0.0) + { + true => Ok((own, raw_data)), false => Err(format!( - "its cost per second {cost} is not a finite non-negative number" + "its cost per second {own} + {raw_data} is not a finite non-negative number" )), } } + +#[cfg(test)] +mod tests; diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs new file mode 100644 index 000000000..c64f88ac9 --- /dev/null +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -0,0 +1,216 @@ +//! Tests of [`super::execution_costs`]. + +use std::time::Duration; + +use asap_types::ir::operator::Reduction; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalEdge, + PhysicalProperties, StorageMedium, +}; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SketchParams, SummaryUpdate}; +use asap_types::test_support::memo_builders::{group, query, scan_group, try_candidate}; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, + TimestampMs, +}; + +use super::*; +use crate::retention::edge_retention_need_ms; +use crate::selectable::{QueryId, SelectableEdge, SelectableExecutions}; + +const DAY_MS: u64 = 86_400_000; + +fn range(days: u64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(days * DAY_MS), + kind: TimeRangeKind::Range, + child, + }) +} + +/// The group of `operator`, with its own candidate at θ `parameters`. +fn kll_group( + memo: &mut Memo, + operator: Operator, + parameters: &SketchParams, +) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + try_candidate(memo, group, operator, Some(parameters.clone())).unwrap(); + group +} + +/// Example 3a's shape: a KLL state over the last five years of +/// 1,000,000 series sampled every 15 s, once a day, either built over +/// the scan's five years or merged from the windows of a 1-day chain +/// built at ingestion time; a second daily query reads the chain's +/// newest day of windows. The scan has a query-time candidate, one at +/// ingestion time and one materialized there (C0, C1, C2); the chain +/// (C3) may read C1 or C2, the five-year range (C4) C0 or C2. +/// +/// An ingestion-time consumer reads one batch of the scan, from C1 or +/// C2 alike: the chain's baseline input is that batch, not the five +/// years C2 retains for its query-time consumer. Selecting an +/// execution, one of its edges and the least retention its consumers +/// may need then never costs less than nothing (was −1,356,902/s: the +/// chain costed over five years of rows, the edge from C1 −22,398,978 +/// and a retention cost per ms inflated by those rows). +#[test] +fn no_execution_reading_a_producer_costs_less_than_nothing() { + let kll = ¶meter_list(SketchAlgorithm::Kll)[6]; + let summary_type = FieldDataType::Sketch(SketchAlgorithm::Kll); + let mut memo = Memo::new(&DataWorkload::default()); + let scan = scan_group(&mut memo, "m", &[], vec![]); + let chain = kll_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: summary_type.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: DAY_MS, + }), + kll, + ); + let five_years = 5 * 365; + // A `TimeRange` over the chain, and the merge over it, carry its θ. + let windows = kll_group(&mut memo, range(five_years, chain), kll); + let newest_day = kll_group(&mut memo, range(1, chain), kll); + let last_five_years = group(&mut memo, range(five_years, scan)); + let state = kll_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: last_five_years, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + kll, + ); + try_candidate( + &mut memo, + state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + Some(kll.clone()), + ) + .unwrap(); + for root in [state, newest_day] { + query(&mut memo, root, None); + } + use ExecutionTiming::{IngestionTime, QueryTime}; + let materialized = Lifetime::AcrossEvaluations(StorageMedium::Memory); + let candidate = |group, index_in_group, timing, lifetime| PhysicalCandidate { + logical_candidate_id: CandidateForMemoGroupId { + group, + index_in_group, + }, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + }; + let candidates = vec![ + candidate(scan, 0, QueryTime, Lifetime::None), + candidate(scan, 0, IngestionTime, Lifetime::None), + candidate(scan, 0, IngestionTime, materialized), + candidate(chain, 0, IngestionTime, materialized), + candidate(last_five_years, 0, QueryTime, Lifetime::None), + candidate(windows, 0, QueryTime, Lifetime::None), + candidate(state, 0, QueryTime, Lifetime::None), + candidate(state, 1, QueryTime, Lifetime::None), + candidate(newest_day, 0, QueryTime, Lifetime::None), + ]; + let edge = |u: usize, c: usize, admissible: bool| PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(u), + consumer_candidate_id: PhysicalCandidateId(c), + input_index_of_consumer: 0, + satisfies_logical_requirement: true, + satisfies_physical_requirement: admissible, + }; + let edges = vec![ + edge(0, 3, false), + edge(1, 3, true), + edge(2, 3, true), + edge(0, 4, true), + edge(1, 4, false), + edge(2, 4, true), + edge(3, 5, true), + edge(3, 8, true), + edge(4, 6, true), + edge(5, 7, true), + ]; + let groups = memo.groups().len(); + let physical = PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group: vec![None; groups], + exact_only_groups_of_query: vec![], + }, + candidates, + edges, + vec![], + ); + let daily = QueryWorkloadEntry { + query: Query("daily".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::default(), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(DAY_MS as u32), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, + }; + let entries = vec![daily.clone(), daily]; + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival": {"ingesting": {"rows_per_second": 1_000_000.0 / 15.0}}, + "sources": {"m": {"series_count": 1_000_000, "sample_interval": 15_000}} + })) + .unwrap(); + let selectable = SelectableExecutions::new( + &physical, + &entries, + &[(QueryId(0), state), (QueryId(1), newest_day)], + ); + let costs = execution_costs( + &physical, + &selectable, + &entries, + &data, + &CostInputs::default(), + ); + // π_v times the least any consumer of `v` reads back, the least + // retention (R) lets a plan selecting `v` keep. + let least_retention = |v: PhysicalCandidateId| { + let least_need_ms = selectable + .edges() + .iter() + .filter(|edge| selectable.executions()[edge.producer].candidate == v) + .filter_map(|edge| edge_retention_need_ms(&physical, &entries, edge.edge)) + .min() + .unwrap_or(0); + costs.retention_cost_per_second(v, least_need_ms) + }; + let mut checked = 0; + for (index, &SelectableEdge { consumer, edge, .. }) in selectable.edges().iter().enumerate() { + let (Ok(execution), Some(edge_cost)) = + (&costs.per_execution[consumer], costs.per_edge[index]) + else { + continue; + }; + let consumer = selectable.executions()[consumer].candidate; + let total = execution + edge_cost + least_retention(consumer); + assert!( + total >= 0.0, + "{consumer:?} reading through {edge:?}: {execution} + {edge_cost} + retention = {total}" + ); + checked += 1; + } + // Among them the chain reading either scan. + assert!(checked >= 2, "{checked} edges costed"); +} diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index a7cc68613..96d153d83 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -4,17 +4,22 @@ //! 4 and 5). //! //! [`select_plan`] reads Stage 2's physical candidates $\mathcal C$ and the -//! selectable part of the physical edges $\mathcal E$ ([`selectable`]), -//! costs them with `asap-cost` ([`cost`]), selects with a MILP over $x_v$, -//! $y_{u,c,i}$, $z_{q,v}$, $t_v$ and $r_s$ ([`milp`]; constraints (A)–(F) -//! and the one-consumer rule; (G) and (H) hold because $y$ exists only on -//! admissible edges), and builds the plan (`PhysicalASAPDAG`) from -//! $\{x = 1\}$ and $\{y = 1\}$ as its last step ([`plan`], with each -//! node's retention from [`retention`]). The extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, -//! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ -//! (section 5), and accuracy at each query root, constraint (K), with -//! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from -//! `asap-cost`. +//! admissible part of the physical edges $\mathcal E$, finds every +//! execution of a candidate in an execution context $\kappa$ and every edge +//! between executions the plan may select ([`selectable`]), costs them with +//! `asap-cost` ([`cost`]), and selects with a MILP ([`milp`]) over +//! $x_{\kappa,v}$ per execution (an `AcrossEvaluations` candidate runs +//! once, $x^{\mathrm{mat}}_v$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ and +//! the retention $h_v$, under constraints (A)–(F) per context, the +//! one-consumer rule, (R) ((G) and (H) hold because $y$ exists only on +//! admissible edges whose producer delivers on the consumer's context) and +//! (X): a query reads the groups Pass 2 lists as exact-only for it through +//! exact candidates only, $a_{q,e}$ marking what its selected path reads. It +//! builds the plan (`PhysicalASAPDAG`) from the selected executions and +//! edges as its last step ([`plan`], each node retaining its candidate's +//! $h_v$, the needs from [`retention`]). Accuracy at each query root, +//! constraint (K), with per-candidate error bounds $E_c$ and $D_c$ +//! (section 4), is the remaining extension. Costs come from `asap-cost`. mod cost; mod milp; @@ -25,14 +30,14 @@ mod selectable; use asap_cost::{request_overhead_cost_per_second, CostInputs, CostModel}; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; -use asap_types::stages::{PhysicalASAPMemoCandidates, PruneReason, PrunedCandidate}; +use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PruneReason, PrunedCandidate}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::cost::selection_costs; +use crate::cost::execution_costs; use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError}; use crate::plan::plan_of; use crate::report::{accepted_accuracy, infeasible}; -use crate::selectable::{output_groups, query_outputs, selectable_edges}; +use crate::selectable::{QueryId, SelectableExecutions}; pub use report::{ large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, @@ -71,54 +76,69 @@ pub fn select_plan( entries: entries.len(), }); } - let selectable = selectable_edges(physical, entries); + let outputs: Vec<(QueryId, MemoGroupId)> = physical + .logical_candidates + .memo + .output_groups() + .into_iter() + .map(|(query, group)| (QueryId(query), group)) + .collect(); + let mut selectable = SelectableExecutions::new(physical, entries, &outputs); let request_overhead_cost_per_second = request_overhead_cost_per_second(entries, cost_inputs) .map_err(|error| ErrorSelectingPlan::Cost { candidate: "every plan (the request overhead)".into(), node: 0, error, })?; - let costs = selection_costs( - physical, - &selectable, - entries, - data, - cost_inputs, - request_overhead_cost_per_second, - ); - let outputs = output_groups(physical); - let query_outputs = query_outputs(physical, entries, &costs, &outputs); - if query_outputs + let costs = execution_costs(physical, &selectable, entries, data, cost_inputs); + // A query output is answered by an execution that could be costed and + // that its query may read (X). + selectable.retain_answering(|query, index, execution| { + costs.per_execution[index].is_ok() + && !physical.logical_candidates.query_must_not_read( + query.0, + physical.physical_candidates[execution.candidate.0].logical_candidate_id, + ) + }); + if selectable + .answering() .iter() - .any(|(_, candidates)| candidates.is_empty()) + .any(|(_, executions)| executions.is_empty()) { return Err(infeasible(&costs)); } let solved = solve_selection(&SelectionInputs { physical, + entries, + selectable: &selectable, costs: &costs, - query_outputs: &query_outputs, }) .map_err(|error| match error { SelectionMilpError::Infeasible => infeasible(&costs), SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), })?; - let plan = plan_of(physical, entries, &solved, &outputs).map_err(|error| { + let plan = plan_of(physical, &solved, &outputs).map_err(|error| { ErrorSelectingPlan::Solver(format!("the selected plan is invalid: {error}")) })?; + // A candidate none of whose executions could be costed. + let mut costed = vec![None; physical.physical_candidates.len()]; + for (execution, cost) in selectable.executions().iter().zip(&costs.per_execution) { + let known = costed[execution.candidate.0].get_or_insert(false); + *known |= cost.is_ok(); + } let selection = Selection { accepted_accuracy_per_query: accepted_accuracy(physical, &plan), - pruned_candidates: costs - .candidate_cost_per_second + pruned_candidates: costed .iter() .zip(&physical.physical_candidates) - .filter(|(cost, _)| cost.is_err()) + .filter(|(costed, _)| **costed == Some(false)) .map(|(_, &candidate)| PrunedCandidate { candidate, reason: PruneReason::UsageNotCostable, }) .collect(), - selected_plan_cost_per_second: solved.objective_cost_per_second, + selected_plan_cost_per_second: solved.cost_per_second_without_request_overhead + + request_overhead_cost_per_second, milp_variable_count: solved.variable_count, milp_constraint_count: solved.constraint_count, milp_construction_ms: solved.construction_ms, diff --git a/crates/stage3-plan-selection/src/milp/dominance.rs b/crates/stage3-plan-selection/src/milp/dominance.rs new file mode 100644 index 000000000..3287909fd --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/dominance.rs @@ -0,0 +1,108 @@ +//! Executions the selection MILP need not choose because another does as +//! well: one with $\ell$ = None whose `OneEvaluation` twin does all it +//! does at no more cost is left out ([`dominated_by_one_evaluation_twin`]). + +use std::collections::{BTreeSet, HashMap}; + +use asap_types::ir::properties::ExecutionTiming; +use asap_types::stages::Lifetime; + +use super::model::SelectionInputs; +use super::COST_RELATIVE_TOLERANCE; +use crate::selectable::QueryId; + +/// Per execution, by position in `SelectableExecutions::executions`, +/// whether it is a query-time execution with $\ell$ = None that its +/// `OneEvaluation` twin $b$ (the same logical candidate, so the same θ and +/// the same error bounds, in the same context) dominates: $b$ costs no +/// more, reads each of its producers on the same input at no more, is +/// read by each of its consumers on the same input at no more, and answers +/// every query output it answers. A plan selecting it selects $b$ in its +/// place at no more cost and stays feasible: $b$'s (F) is the looser +/// $x \le$ uses, its edges are the same ones (so (A), (B), (E) and (X) +/// hold as before), and (K), per execution, sees the same θ through the +/// same producers. A candidate with $h_v$ or raw data on disk is never +/// left out so, its retention depending on its own readers. +/// +/// Example: a `SummaryAgg` KLL build read by one readout in a context, +/// offered with $\ell$ = None and `OneEvaluation` at equal cost: the None +/// execution is left out. +pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> Vec { + let selectable = inputs.selectable; + let physical = inputs.physical; + let costs = inputs.costs; + let executions = selectable.executions(); + let edges = selectable.edges(); + let count = executions.len(); + // No more, to rounding (to 1e-24/s when `than` is 0). + let no_more = + |cost: f64, than: f64| cost <= than + COST_RELATIVE_TOLERANCE * than.abs().max(1e-12); + let mut answers: Vec> = vec![BTreeSet::new(); count]; + for (query, answering) in selectable.answering() { + for &index in answering { + answers[index].insert(*query); + } + } + // The query-time executions of each logical candidate in each context, + // without $h_v$ or raw data and costed: (with ℓ = None, OneEvaluation). + let mut twins: HashMap<_, (Vec, Vec)> = HashMap::new(); + for (index, execution) in executions.iter().enumerate() { + let v = execution.candidate; + let candidate = &physical.physical_candidates[v.0]; + let properties = candidate.physical_properties; + if properties.timing() != ExecutionTiming::QueryTime + || costs.retention_cost_per_retained_second.contains_key(&v) + || costs.raw_data_source.contains_key(&v) + || costs.per_execution[index].is_err() + { + continue; + } + let (none, one_evaluation) = twins + .entry((execution.context, candidate.logical_candidate_id)) + .or_default(); + match properties.lifetime() { + Lifetime::None => none.push(index), + Lifetime::OneEvaluation(_) => one_evaluation.push(index), + Lifetime::AcrossEvaluations(_) => {} + } + } + let cost = |index: usize| *costs.per_execution[index].as_ref().expect("costed"); + let input_of = |position: usize| physical.edge(edges[position].edge).input_index_of_consumer; + // Whether `b` reads each producer `a` reads on an input, and is read by + // each consumer that reads `a` on its input, over a costed edge at no + // more. + let edges_no_dearer = |a: usize, b: usize| { + let covered = |position: usize, twins: &[usize], same_end: &dyn Fn(usize) -> bool| { + let Some(edge_cost) = costs.per_edge[position] else { + return true; + }; + twins.iter().any(|&twin| { + same_end(twin) + && costs.per_edge[twin].is_some_and(|twin_cost| no_more(twin_cost, edge_cost)) + }) + }; + (0..selectable.input_count(a)).all(|input| { + selectable.edges_into(a, input).iter().all(|&position| { + covered(position, selectable.edges_into(b, input), &|twin| { + edges[twin].producer == edges[position].producer + }) + }) + }) && selectable.edges_out_of(a).iter().all(|&position| { + covered(position, selectable.edges_out_of(b), &|twin| { + edges[twin].consumer == edges[position].consumer + && input_of(twin) == input_of(position) + }) + }) + }; + let mut dominated = vec![false; count]; + for (none, one_evaluation) in twins.values() { + for &a in none { + dominated[a] = one_evaluation.iter().any(|&b| { + no_more(cost(b), cost(a)) + && edges_no_dearer(a, b) + && answers[a].is_subset(&answers[b]) + }); + } + } + dominated +} diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs new file mode 100644 index 000000000..f48352bb0 --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -0,0 +1,340 @@ +//! Symmetry reduction of the selection MILP: a copy of an equivalent +//! evaluation context shares the original's variables +//! ([`copies_in_equivalent_contexts`]), so HiGHS does not branch over each +//! copy. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use asap_types::accuracy::AccuracyTarget; +use asap_types::stages::{ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId}; + +use super::model::SelectionInputs; +use super::COST_RELATIVE_TOLERANCE; +use crate::selectable::QueryId; + +/// Per execution, edge (by position) and query, the one whose MILP +/// variables it takes: itself unless it is in a copy of an equivalent +/// context ([`copies_in_equivalent_contexts`]). +/// +/// Example: the same query every 60 s (Q0) and every 300 s (Q1): each +/// execution and edge of the 300 s context maps to its counterpart in the +/// 60 s one, and `original_of_query` is `[Q0, Q0]`. +pub(crate) struct SharedVariables { + /// By position in `SelectableExecutions::executions`. + pub(crate) original_of_execution: Vec, + /// By position in `SelectableExecutions::edges`. + pub(crate) original_of_edge: Vec, + /// By [`QueryId`]. + pub(crate) original_of_query: Vec, +} + +/// An execution as a root evaluation context sees it: its own, by position +/// among the context's executions, or one outside every evaluation context +/// (`Materialized(v)` or an ingestion pipeline), by position in +/// `SelectableExecutions::executions`, which every context reading it +/// shares. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +enum ExecutionInContext { + Own(usize), + Shared(usize), +} + +/// What two equivalent contexts have alike, up to their costs' common +/// multiple $\alpha$: per execution (ascending, so by candidate) its +/// candidate, whether it is kept and costed, its least retention, and +/// per input the memo edges into it from each producer, costed or not; and +/// the sorted signatures of its queries ([`QuerySignature`]). +#[derive(Debug, PartialEq, Eq, Hash)] +struct ContextKey<'a> { + executions: Vec, + queries: Vec>, +} + +/// One execution's part of a [`ContextKey`]. +#[derive(Debug, PartialEq, Eq, Hash)] +struct ExecutionKey { + candidate: PhysicalCandidateId, + kept: bool, + costed: bool, + least_retention_ms: Option, + /// Per input, each edge into it: (memo edge, producer, costed). + inputs: Vec>, +} + +/// A query's part of a [`ContextKey`]. +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct QuerySignature<'a> { + /// Per output, the executions that may answer it. + answering_executions_per_output: Vec>, + /// The groups it may read only exactly. + exact_only_groups: Option<&'a BTreeSet>, + /// Its range-evaluation span (`QueryWorkloadEntry::range_span_ms`). + range_span_ms: u64, + /// Its accuracy target, by index among the workload's distinct targets + /// ([`AccuracyTarget`] holds floats: metric, ε, δ and scope all equal). + accuracy_target_index: usize, +} + +/// Which executions, edges and queries share variables +/// ([`SharedVariables`]): a root evaluation context (no request anchor) +/// equivalent to an earlier one is a copy of it, and so are its +/// executions, edges and queries. Two such contexts are equivalent when +/// they have the same [`ContextKey`] (the same candidates, kept alike, the +/// same memo edges from producers that correspond, in the context or the +/// same execution outside every evaluation context, and queries that pair +/// up with the same output executions, exact-only groups, span and +/// accuracy target, so the accuracy constraint holds alike in both) and +/// every cost the MILP reads in one (of a kept execution, or of an edge +/// between two kept ones) is the same multiple $\alpha > 0$ of the +/// other's, to [`COST_RELATIVE_TOLERANCE`] of the copy's largest such +/// cost. Each query belongs to its own evaluation context +/// (`SelectableExecutions::evaluation_context`), wherever its answering +/// executions run. Contexts are bucketed by key, so only the cost check +/// compares two contexts. +/// +/// Why some optimum survives: from an optimal plan, give both contexts the +/// selection of the one whose own cost $A$ or $B$ is lower, say $A$. The +/// copied context then costs $\alpha A \le \alpha B$, the retention $h_v$ +/// and raw data $r_s$ (each the most of their readers' needs, equal in both +/// copies) do not grow, and an execution outside them that only the other +/// selection read is left out at no cost; with $B \le A$ symmetrically. So +/// some optimum selects alike in both, and sharing their variables keeps +/// it, to the tolerance the costs are compared to: each copied cost the +/// MILP reads (of a kept execution, or of an edge between two kept ones) +/// within 1e-12 of the largest of them of $\alpha$ times its original. The +/// exchange needs $\alpha > 0$. +/// +/// Example: the same queries every 60 s and every 300 s (one user each): +/// the 300 s context's executions cost 1/5 of the 60 s ones' and share +/// their variables. +pub(crate) fn copies_in_equivalent_contexts( + inputs: &SelectionInputs<'_>, + kept: &[bool], +) -> SharedVariables { + let selectable = inputs.selectable; + let costs = inputs.costs; + let executions = selectable.executions(); + let edges = selectable.edges(); + let mut shared = SharedVariables { + original_of_execution: (0..executions.len()).collect(), + original_of_edge: (0..edges.len()).collect(), + original_of_query: (0..inputs.entries.len()).map(QueryId).collect(), + }; + // Each root evaluation context's executions, ascending (so by + // candidate), and each execution's position among them. + let mut executions_of: BTreeMap> = BTreeMap::new(); + let mut position_in_context: Vec> = vec![None; executions.len()]; + for (index, execution) in executions.iter().enumerate() { + if selectable.is_root_evaluation_context(execution.context) { + let own = executions_of.entry(execution.context).or_default(); + position_in_context[index] = Some(own.len()); + own.push(index); + } + } + let mut outputs_of: BTreeMap> = BTreeMap::new(); + for (query, answering) in selectable.answering() { + outputs_of.entry(*query).or_default().push(answering); + } + // Each evaluation context's queries; a query without an output has + // none. + let mut queries_of: BTreeMap> = BTreeMap::new(); + for query in (0..inputs.entries.len()).map(QueryId) { + if let Some(context) = selectable.evaluation_context(query) { + queries_of.entry(context).or_default().push(query); + } + } + // `index` as `context` sees it; `None` for one in another evaluation + // context (a request context), which no copy shares. + let in_context = |context: ExecutionContextId, index: usize| { + let execution_context = executions[index].context; + if execution_context == context { + position_in_context[index].map(ExecutionInContext::Own) + } else { + selectable + .context(execution_context) + .is_outside_evaluations() + .then_some(ExecutionInContext::Shared(index)) + } + }; + // Each query's accuracy target, as an index among the distinct ones. + let mut targets: Vec<&AccuracyTarget> = Vec::new(); + let target_of_query: Vec = inputs + .entries + .iter() + .map(|entry| { + let target = &entry.requirements.accuracy; + targets + .iter() + .position(|&known| known == target) + .unwrap_or_else(|| { + targets.push(target); + targets.len() - 1 + }) + }) + .collect(); + let signature = |context: ExecutionContextId, query: QueryId| -> Option> { + // A query has an evaluation context exactly when it has an output reference. + let outputs = outputs_of[&query] + .iter() + .map(|answering| { + answering + .iter() + .map(|&index| in_context(context, index)) + .collect::>>() + }) + .collect::>>()?; + Some(QuerySignature { + answering_executions_per_output: outputs, + exact_only_groups: inputs + .physical + .logical_candidates + .exact_only_groups_of_query + .get(query.0), + range_span_ms: inputs.entries[query.0].range_span_ms(), + accuracy_target_index: target_of_query[query.0], + }) + }; + // `context`'s key, and its queries in the order of their signatures in + // it. + let key_of = |context: ExecutionContextId| -> Option<(ContextKey<'_>, Vec)> { + let executions = executions_of[&context] + .iter() + .map(|&index| { + let inputs = (0..selectable.input_count(index)) + .map(|input| { + selectable + .edges_into(index, input) + .iter() + .map(|&position| { + let edge = edges[position]; + Some(( + edge.edge, + in_context(context, edge.producer)?, + costs.per_edge[position].is_some(), + )) + }) + .collect::>>() + }) + .collect::>>()?; + Some(ExecutionKey { + candidate: executions[index].candidate, + kept: kept[index], + costed: costs.per_execution[index].is_ok(), + least_retention_ms: costs.least_retention_ms_per_execution[index], + inputs, + }) + }) + .collect::>>()?; + let mut queries = queries_of + .get(&context) + .into_iter() + .flatten() + .map(|&query| Some((signature(context, query)?, query))) + .collect::>>()?; + queries.sort(); + let (queries, in_signature_order) = queries.into_iter().unzip(); + Some(( + ContextKey { + executions, + queries, + }, + in_signature_order, + )) + }; + // Whether every cost of `copy` the MILP reads is the same multiple + // α > 0 of `original`'s, their executions and edges paired in order + // (equal keys), to `COST_RELATIVE_TOLERANCE` of the copy's largest such + // cost, α taken from that pair. Only kept executions and edges between + // two kept ones count (equal keys keep them alike in both): a left-out + // execution has no cost in the MILP, and one costing 1e12 would + // otherwise allow each kept cost an error of 1. An edge cost is the consumer's cost through one producer + // less its own, a small difference of large costs that keeps their + // rounding: 2.8110024e-9 − 2.8110047e-9 in a 60 s context and the same + // over 5 in a 300 s one are −2.3556e-15 and −4.7112e-16, a ratio of + // 0.200000000026, off by far more than 1e-12 of themselves but not of + // the context's costs. + let proportional_costs = |original: ExecutionContextId, copy: ExecutionContextId| { + let mut pairs: Vec<(f64, f64)> = Vec::new(); + for (&a, &b) in executions_of[&original].iter().zip(&executions_of[©]) { + if !kept[a] { + continue; + } + if let (Ok(cost_a), Ok(cost_b)) = (&costs.per_execution[a], &costs.per_execution[b]) { + pairs.push((*cost_a, *cost_b)); + } + for input in 0..selectable.input_count(a) { + for (&edge_a, &edge_b) in selectable + .edges_into(a, input) + .iter() + .zip(selectable.edges_into(b, input)) + { + if !kept[edges[edge_a].producer] { + continue; + } + if let (Some(cost_a), Some(cost_b)) = + (costs.per_edge[edge_a], costs.per_edge[edge_b]) + { + pairs.push((cost_a, cost_b)); + } + } + } + } + let Some(&(largest_a, largest_b)) = + pairs.iter().max_by(|x, y| x.1.abs().total_cmp(&y.1.abs())) + else { + return true; + }; + if largest_a == 0.0 || largest_b == 0.0 { + return pairs.iter().all(|&(a, b)| a == 0.0 && b == 0.0); + } + let alpha = largest_b / largest_a; + if alpha <= 0.0 { + return false; + } + let tolerance = COST_RELATIVE_TOLERANCE * largest_b.abs(); + pairs + .iter() + .all(|&(a, b)| (b - alpha * a).abs() <= tolerance) + }; + // Per key, the originals, each with its queries in signature order. + let mut originals_of_key: HashMap, Vec<(ExecutionContextId, Vec)>> = + HashMap::new(); + for © in executions_of.keys() { + let Some((key, copy_queries)) = key_of(copy) else { + continue; + }; + let originals = originals_of_key.entry(key).or_default(); + let Some((original, original_queries)) = originals + .iter() + .find(|(original, _)| proportional_costs(*original, copy)) + else { + originals.push((copy, copy_queries)); + continue; + }; + let original = *original; + for (&a, &b) in executions_of[&original].iter().zip(&executions_of[©]) { + shared.original_of_execution[b] = a; + for input in 0..selectable.input_count(b) { + for (&edge_a, &edge_b) in selectable + .edges_into(a, input) + .iter() + .zip(selectable.edges_into(b, input)) + { + shared.original_of_edge[edge_b] = edge_a; + } + } + // Implied by the key: an execution of a root context is read + // only by consumers in that context, whose in-edges the key + // pairs. + debug_assert_eq!( + selectable.edges_out_of(a).len(), + selectable.edges_out_of(b).len() + ); + } + // Equal keys: the queries pair up in signature order. + for (&a, b) in original_queries.iter().zip(copy_queries) { + shared.original_of_query[b.0] = a; + } + } + shared +} diff --git a/crates/stage3-plan-selection/src/milp/exact_only.rs b/crates/stage3-plan-selection/src/milp/exact_only.rs new file mode 100644 index 000000000..88b176b15 --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/exact_only.rs @@ -0,0 +1,128 @@ +//! Exact-only reads per query +//! (`docs/design_docs/concepts/accuracy-models.md`, "the groups it may read +//! only exactly"): a query reads the groups Pass 2 lists as exact-only for +//! it (`LogicalASAPMemoCandidates::exact_only_groups_of_query`) through +//! exact candidates only, in its own selection context, while another query +//! may still read a sketch of those groups through its own path. +//! +//! $a_{q,e} \in [0, 1]$ marks the executions $e$ query $q$'s selected path +//! reads, from its output down; constraint (X) keeps it at 0 on every +//! execution $q$ must not read. + +use std::collections::{BTreeMap, BTreeSet}; + +use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; + +use super::model::SelectionInputs; +use crate::selectable::QueryId; + +/// $a_{q,e}$, added to `problem`, for each query $q$ and each execution $e$ +/// (by position in `SelectableExecutions::executions`) it may reach from +/// its output executions over selectable edges (`y`) from which an +/// execution it must not read +/// (`LogicalASAPMemoCandidates::query_must_not_read`) is reachable, less those +/// themselves. None for a query with no exact-only group. +pub(crate) fn exact_only_variables( + inputs: &SelectionInputs<'_>, + y: &BTreeMap, + problem: &mut ProblemVariables, +) -> BTreeMap<(QueryId, usize), Variable> { + let selectable = inputs.selectable; + let count = selectable.executions().len(); + let edges = selectable.edges(); + // The selectable edges (those with a `y`), producers per consumer and + // back. + let producers = |e: usize| { + (0..selectable.input_count(e)) + .flat_map(move |input| selectable.edges_into(e, input)) + .filter(|position| y.contains_key(position)) + .map(|&position| edges[position].producer) + }; + let consumers = |e: usize| { + selectable + .edges_out_of(e) + .iter() + .filter(|position| y.contains_key(position)) + .map(|&position| edges[position].consumer) + }; + let forbidden = |query: QueryId, e: usize| { + let v = selectable.executions()[e].candidate; + inputs.physical.logical_candidates.query_must_not_read( + query.0, + inputs.physical.physical_candidates[v.0].logical_candidate_id, + ) + }; + let queries: BTreeSet = selectable.answering().iter().map(|(q, _)| *q).collect(); + let mut tracked = BTreeMap::new(); + for query in queries { + let mut reached = vec![false; count]; + let mut pending: Vec = selectable + .answering() + .iter() + .filter(|(q, _)| *q == query) + .flat_map(|(_, executions)| executions.iter().copied()) + .collect(); + while let Some(e) = pending.pop() { + if !std::mem::replace(&mut reached[e], true) { + pending.extend(producers(e)); + } + } + let mut reaches_forbidden = vec![false; count]; + let mut pending: Vec = (0..count) + .filter(|&e| reached[e] && forbidden(query, e)) + .collect(); + while let Some(e) = pending.pop() { + if reached[e] && !std::mem::replace(&mut reaches_forbidden[e], true) { + pending.extend(consumers(e)); + } + } + for e in (0..count).filter(|&e| reaches_forbidden[e] && !forbidden(query, e)) { + tracked.insert((query, e), problem.add(variable().min(0).max(1))); + } + } + tracked +} + +/// (X) Query $q$ reads the groups it may read only exactly through exact +/// candidates: $a_{q,e} \ge z_{q,e}$ for an output execution, +/// $a_{q,p} \ge y_{\kappa,p,c,i} + a_{q,c} - 1$ along a selectable edge, so +/// $a_{q,e} = 1$ on every execution $q$'s selected path reads; along an +/// edge into an execution $p$ $q$ must not read +/// (`LogicalASAPMemoCandidates::query_must_not_read`), +/// $y_{\kappa,p,c,i} + a_{q,c} \le 1$. No $z_{q,e}$ exists for an output +/// execution $q$ must not read (`crate::select_plan` leaves it out). Other +/// queries may still read it, through their own paths. $a$ exists only +/// where [`exact_only_variables`] tracks it, and is not in the objective. +pub(crate) fn exact_only_constraints( + inputs: &SelectionInputs<'_>, + a: &BTreeMap<(QueryId, usize), Variable>, + y: &BTreeMap, + z: &BTreeMap<(QueryId, usize), Variable>, +) -> Vec { + let selectable = inputs.selectable; + let mut constraints: Vec = z + .iter() + .filter_map(|(key, &z)| Some(constraint!(*a.get(key)? >= z))) + .collect(); + // The tracked queries, per execution. + let mut readers: BTreeMap> = BTreeMap::new(); + for (&(query, execution), &a) in a { + readers.entry(execution).or_default().push((query, a)); + } + for (&position, &y) in y { + let edge = selectable.edges()[position]; + let (consumer, producer) = (edge.consumer, edge.producer); + for &(query, a_consumer) in readers.get(&consumer).into_iter().flatten() { + let v = selectable.executions()[producer].candidate; + if inputs.physical.logical_candidates.query_must_not_read( + query.0, + inputs.physical.physical_candidates[v.0].logical_candidate_id, + ) { + constraints.push(constraint!(y + a_consumer <= 1)); + } else if let Some(&a_producer) = a.get(&(query, producer)) { + constraints.push(constraint!(a_producer >= y + a_consumer - 1)); + } + } + } + constraints +} diff --git a/crates/stage3-plan-selection/src/milp/execution_contexts.rs b/crates/stage3-plan-selection/src/milp/execution_contexts.rs deleted file mode 100644 index e1b66418f..000000000 --- a/crates/stage3-plan-selection/src/milp/execution_contexts.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! The execution-context extension of the selection MILP -//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 5): its variables and constraint (R), `todo!()` until execution -//! contexts are selected. - -use std::collections::HashMap; - -use good_lp::{Constraint, Variable}; - -use asap_types::stages::{ - ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, -}; - -/// The execution-context variables (section 5), which replace $x$ and the -/// unindexed $y$ of the baseline. -#[allow(dead_code)] // Read once execution contexts are selected. -struct ExecutionContextVariables { - /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. - x_materialized: HashMap, - /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. - x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, - /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. - y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, - /// $h_v$, for an `AcrossEvaluations` producer. - retention_amount: HashMap, -} - -/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$: for a -/// time-indexed producer (raw data, a chain, results stored per evaluation -/// time) the reading path's offset plus range, a constant per edge, so (R) -/// is linear. Query latency is ignored: a result counts as read when its -/// evaluation starts. -#[allow(dead_code)] // Called once execution contexts are selected. -fn retention_constraints( - physical: &PhysicalASAPMemoCandidates, - variables: &ExecutionContextVariables, -) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 7: constraint (R)") -} diff --git a/crates/stage3-plan-selection/src/milp/graph.rs b/crates/stage3-plan-selection/src/milp/graph.rs index ec2da8fd5..3bbaa2168 100644 --- a/crates/stage3-plan-selection/src/milp/graph.rs +++ b/crates/stage3-plan-selection/src/milp/graph.rs @@ -1,33 +1,44 @@ //! The strongly connected components of the selectable edges between -//! candidates: acyclicity (E) needs an order variable $t_v$ only inside a +//! executions: acyclicity (E) needs an order variable $t$ only inside a //! component, bounded by its size. use std::collections::HashMap; use super::model::SelectionInputs; -/// Each candidate's strongly connected component of the selectable edges -/// (those [`crate::cost::SelectionCosts::edge_cost_change_per_second`] costs), computed once per model. +/// Each execution's strongly connected component of the selectable edges +/// (those [`crate::cost::ExecutionCosts::per_edge`] costs), computed once +/// per model, by position in `SelectableExecutions::executions`. pub(crate) struct Components { - /// Each candidate's component number. + /// Each execution's component number. pub(crate) of: Vec, - /// The size of each candidate's component: 1 for a candidate on no + /// The size of each execution's component: 1 for an execution on no /// cycle. pub(crate) size: Vec, } impl Components { pub(crate) fn of(inputs: &SelectionInputs<'_>) -> Self { + let selectable = inputs.selectable; + let edges = selectable.edges(); + let costed = |position: &&usize| inputs.costs.per_edge[**position].is_some(); let of = strongly_connected_components( - inputs.physical.physical_candidates.len(), - inputs - .costs - .edge_cost_change_per_second - .keys() - .map(|&edge| { - let edge = inputs.physical.edge(edge); - (edge.producer_candidate_id.0, edge.consumer_candidate_id.0) - }), + selectable.executions().len(), + |producer| { + selectable + .edges_out_of(producer) + .iter() + .filter(costed) + .map(|&position| edges[position].consumer) + .collect() + }, + |consumer| { + (0..selectable.input_count(consumer)) + .flat_map(|input| selectable.edges_into(consumer, input)) + .filter(costed) + .map(|&position| edges[position].producer) + .collect() + }, ); let mut count: HashMap = HashMap::new(); for &component in &of { @@ -38,18 +49,14 @@ impl Components { } } -/// Each node's strongly connected component of the graph with `arcs` -/// (Kosaraju), as a component number. +/// Each node's strongly connected component of the graph whose arcs out of +/// a node are `forward(node)` and into it `backward(node)` (Kosaraju), as +/// a component number. fn strongly_connected_components( node_count: usize, - arcs: impl Iterator, + forward: impl Fn(usize) -> Vec, + backward: impl Fn(usize) -> Vec, ) -> Vec { - let mut forward = vec![Vec::new(); node_count]; - let mut backward = vec![Vec::new(); node_count]; - for (from, to) in arcs { - forward[from].push(to); - backward[to].push(from); - } // Nodes by finishing time of a depth-first search over `forward`. let mut finished = Vec::with_capacity(node_count); let mut visited = vec![false; node_count]; @@ -57,12 +64,12 @@ fn strongly_connected_components( if std::mem::replace(&mut visited[start], true) { continue; } - let mut stack = vec![(start, 0)]; - while let Some((node, next)) = stack.pop() { - if let Some(&to) = forward[node].get(next) { - stack.push((node, next + 1)); + let mut stack = vec![(start, forward(start), 0)]; + while let Some((node, out, next)) = stack.pop() { + if let Some(&to) = out.get(next) { + stack.push((node, out, next + 1)); if !std::mem::replace(&mut visited[to], true) { - stack.push((to, 0)); + stack.push((to, forward(to), 0)); } } else { finished.push(node); @@ -77,7 +84,7 @@ fn strongly_connected_components( component[start] = number; let mut stack = vec![start]; while let Some(node) = stack.pop() { - for &from in &backward[node] { + for from in backward(node) { if component[from] == usize::MAX { component[from] = number; stack.push(from); diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 3b30ee22c..32e402d83 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -1,11 +1,27 @@ //! Stage 3's selection MILP //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! sections 3–5): the baseline's variables and one function per constraint -//! family ([`model`]), the graph algorithm it needs ([`graph`]), the bound -//! it scales by ([`upper_bound`]), and the execution-context and accuracy -//! extensions ([`execution_contexts`], [`accuracy`]). +//! sections 3–5): its variables over execution contexts and one function +//! per constraint family ([`model`]), the retention amount $h_v$ and +//! constraint (R) ([`retention`]), each query's exact-only reads, +//! constraint (X) ([`exact_only`]), the executions other executions +//! dominate ([`dominance`]), the copies of equivalent contexts that share +//! variables ([`equivalent_contexts`]), the graph algorithm it needs ([`graph`]), +//! the bound that leaves out executions dearer than a feasible plan before +//! scaling ([`upper_bound`]), and the accuracy extension ([`accuracy`]). mod accuracy; -mod execution_contexts; +mod dominance; +mod equivalent_contexts; +pub(crate) mod exact_only; mod graph; pub(crate) mod model; +mod retention; mod upper_bound; + +/// The relative tolerance to which two costs per second are taken as +/// equal (or one as no more than the other) when the MILP is reduced +/// before it is built: costs are rates times costs per run, so equal costs +/// can differ in their last bits. +/// +/// Example: 0.1 × 3.0 and 0.3 differ by about 5.6e-17, well within +/// 1e-12 of 0.3. +const COST_RELATIVE_TOLERANCE: f64 = 1e-12; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index ce2d5c7cf..1394a0982 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -4,15 +4,17 @@ //! //! | Family | Variables | Constraints | //! |---|---|---| -//! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_constraints`]; (G), (H): $y$ exists only on admissible edges | -//! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ (`ExecutionContextVariables`) | (R) `retention_constraints` | +//! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_rows`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | +//! | retention (section 5) | $h_v$ | (R) [`super::retention::retention_constraints`] | +//! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | //! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | //! -//! The baseline is solved by [`solve_selection`]; the extensions' -//! variables and constraints are in [`super::execution_contexts`] and -//! [`super::accuracy`], `todo!()` until they are added. The strongly -//! connected components (E) needs are in [`super::graph`]; the bound that -//! leaves out candidates before scaling in [`super::upper_bound`]. +//! [`solve_selection`] solves it without (K), whose variables and +//! constraint are in [`super::accuracy`], `todo!()` until it is added. The +//! strongly connected components (E) needs are in [`super::graph`]; the +//! executions left out before scaling, those dearer than a feasible plan +//! and those their `OneEvaluation` twin dominates, in +//! [`super::upper_bound`] and [`super::dominance`]. use std::collections::{BTreeMap, HashMap}; use std::time::Instant; @@ -23,132 +25,177 @@ use good_lp::{ }; use asap_types::ir::operator::Source; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::Lifetime; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; +use asap_types::workload::QueryWorkloadEntry; +use super::dominance::dominated_by_one_evaluation_twin; +use super::equivalent_contexts::{copies_in_equivalent_contexts, SharedVariables}; +use super::exact_only::{exact_only_constraints, exact_only_variables}; use super::graph::Components; +use super::retention::{least_retention_ms, retention_constraints, retention_variables}; use super::upper_bound::dearer_than_a_feasible_plan; -use crate::cost::SelectionCosts; +use crate::cost::ExecutionCosts; +use crate::selectable::{Execution, QueryId, SelectableExecutions}; -/// A query, by index in the workload. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct QueryId(pub usize); - -/// What the baseline MILP is built over. +/// What the MILP is built over. pub(crate) struct SelectionInputs<'a> { pub(crate) physical: &'a PhysicalASAPMemoCandidates, - /// $w_v$ and $c_{u,c,i}$: $y$ exists only on the edges - /// [`SelectionCosts::edge_cost_change_per_second`] costs, $x_v$ is 0 for a - /// candidate it could not cost. - pub(crate) costs: &'a SelectionCosts, - /// Each query output reference (a query, or one operator a scalar query - /// reads), with the candidates that may answer it: $z_{q,v}$ exists - /// only for them. - pub(crate) query_outputs: &'a [(QueryId, Vec)], + /// Each query's workload entry, for the retention needs of (R). + pub(crate) entries: &'a [QueryWorkloadEntry], + /// The executions $x$, edges $y$ and query outputs $z$ are over: + /// $z_{q,v}$ exists only for the executions + /// `SelectableExecutions::answering` lists. + pub(crate) selectable: &'a SelectableExecutions, + /// The objective's coefficients: $x$ is 0 for an execution it could + /// not cost, $y$ exists only on an edge it costs, $h_v$ only for a + /// candidate with a retention cost. + pub(crate) costs: &'a ExecutionCosts, } -/// The baseline variables (section 3). +/// The variables (sections 3 and 5). Executions are by position in +/// `SelectableExecutions::executions`, edges by position in +/// `SelectableExecutions::edges`. pub(crate) struct SelectionVariables { - /// $x_v$, by [`PhysicalCandidateId`]. + /// $x_{\kappa,v}$, by execution. x: Vec, - /// $y_{u,c,i}$, on admissible edges only, in edge order. - y: BTreeMap, - /// $z_{q,v}$, for $v \in G_q$, in key order. - z: BTreeMap<(QueryId, PhysicalCandidateId), Variable>, - /// $t_v$, by [`PhysicalCandidateId`]: only for candidates on a cycle of - /// admissible edges (a strongly connected component); `None` elsewhere. + /// $y_{\kappa,u,c,i}$, by edge, on the edges costed between executions + /// kept, in edge order. + y: BTreeMap, + /// $z_{q,v}$, by query and the execution answering it, in key order. + z: BTreeMap<(QueryId, usize), Variable>, + /// $t$, by execution: only for executions on a cycle of selectable + /// edges (a strongly connected component); `None` elsewhere. t: Vec>, + /// $h_v$ in seconds retained, for a candidate with a retention cost and + /// an execution kept ([`retention_variables`]). + h: BTreeMap, /// Per raw-data source a query-time scan keeps on disk at a cost: the - /// cost per second of keeping it, at least each selected scan's - /// ([`raw_data_disk_constraints`]). + /// cost per second of keeping it, at least each selected scan's, + /// scaled by the objective's scale ([`raw_data_disk_rows`]). raw_data_on_disk: BTreeMap, + /// $a_{q,e}$: query $q$'s selected path reads execution $e$, where + /// (X) needs it ([`exact_only_variables`]). + read_by_query: BTreeMap<(QueryId, usize), Variable>, } -/// $x$, $y$, $z$, $t$ and $r$ over `inputs`, added to `problem`: $x_v$ -/// fixed at 0 for a candidate that could not be costed or is `left_out`, -/// and no $y$ on an edge of one. +/// $x$, $y$, $z$, $t$, $h$ and $r$ over `inputs`, added to `problem`: $x$ +/// fixed at 0 for an execution that is not `kept` (not costed, or dearer +/// than a feasible plan), and no $y$ on an edge of one. The executions, +/// edges and queries of a copy of an equivalent evaluation context take the +/// original's $x$, $y$ and $z$ ([`copies_in_equivalent_contexts`]), so +/// their costs add up on one variable. fn selection_variables( inputs: &SelectionInputs<'_>, - left_out: &[bool], + kept: &[bool], components: &Components, problem: &mut ProblemVariables, ) -> SelectionVariables { - let count = inputs.physical.physical_candidates.len(); - let x = (0..count) - .map( - |v| match inputs.costs.candidate_cost_per_second[v].is_ok() && !left_out[v] { - true => problem.add(variable().binary()), - false => problem.add(variable().min(0).max(0)), - }, - ) - .collect(); - let mut edges: Vec = inputs - .costs - .edge_cost_change_per_second - .keys() - .copied() - .filter(|&edge| { - let edge = inputs.physical.edge(edge); - !left_out[edge.producer_candidate_id.0] && !left_out[edge.consumer_candidate_id.0] + let selectable = inputs.selectable; + let SharedVariables { + original_of_execution, + original_of_edge, + original_of_query, + } = copies_in_equivalent_contexts(inputs, kept); + let mut x: Vec = Vec::with_capacity(kept.len()); + for (index, &kept) in kept.iter().enumerate() { + let variable = match (original_of_execution[index], kept) { + (original, _) if original != index => x[original], + (_, true) => problem.add(variable().binary()), + (_, false) => problem.add(variable().min(0).max(0)), + }; + x.push(variable); + } + let mut y: BTreeMap = BTreeMap::new(); + for (position, (edge, cost)) in selectable + .edges() + .iter() + .zip(&inputs.costs.per_edge) + .enumerate() + { + if cost.is_some() && kept[edge.consumer] && kept[edge.producer] { + let variable = match original_of_edge[position] { + original if original != position => y[&original], + _ => problem.add(variable().binary()), + }; + y.insert(position, variable); + } + } + let mut z = BTreeMap::new(); + #[cfg(debug_assertions)] + let answering_pairs: std::collections::BTreeSet<(QueryId, usize)> = selectable + .answering() + .iter() + .flat_map(|(query, executions)| { + executions.iter().map(move |&execution| (*query, execution)) }) .collect(); - edges.sort_unstable(); - let y = edges - .into_iter() - .map(|edge| (edge, problem.add(variable().binary()))) - .collect(); - let mut z = BTreeMap::new(); - for (query, candidates) in inputs.query_outputs { - for &v in candidates { - z.entry((*query, v)) + for (query, executions) in selectable.answering() { + for &execution in executions { + let original = (original_of_query[query.0], original_of_execution[execution]); + // A shared z is that of a real (query, answering execution) + // pair: a copy's query pairs with an original query of the + // same output executions. + #[cfg(debug_assertions)] + debug_assert!( + answering_pairs.contains(&original), + "z of {query:?} at execution {execution} shares {original:?}, not an answering pair" + ); + let variable = *z + .entry(original) .or_insert_with(|| problem.add(variable().binary())); + z.insert((*query, execution), variable); } } - // t_v only inside a strongly connected component of the selectable + // t only inside a strongly connected component of the selectable // edges, bounded by its size. let t = components .size .iter() .map(|&size| (size > 1).then(|| problem.add(variable().min(0).max((size - 1) as f64)))) .collect(); + let h = retention_variables(inputs, kept, problem); let mut raw_data_on_disk = BTreeMap::new(); - for (source, _) in inputs.costs.raw_data_on_disk.iter().flatten() { + for source in inputs.costs.raw_data_source.values() { raw_data_on_disk .entry(source.clone()) .or_insert_with(|| problem.add(variable().min(0))); } + let read_by_query = exact_only_variables(inputs, &y, problem); SelectionVariables { x, y, z, t, + h, raw_data_on_disk, + read_by_query, } } -/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$. +/// (A) $\sum_{u \in P_{c,i}} y_{\kappa,u,c,i} = x_{\kappa,c}$, per execution +/// of $c$ and input $i$. fn input_satisfaction_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let physical = inputs.physical; + let selectable = inputs.selectable; let mut constraints = Vec::new(); - for c in (0..physical.physical_candidates.len()).map(PhysicalCandidateId) { - for i in 0..physical.input_count(c) { - let producers: Expression = physical - .admissible_edges_into(c, i) + for (index, &x) in variables.x.iter().enumerate() { + for i in 0..selectable.input_count(index) { + let chosen: Expression = selectable + .edges_into(index, i) .iter() - .filter_map(|edge| variables.y.get(edge)) + .filter_map(|position| variables.y.get(position)) .sum(); - constraints.push(constraint!(producers == variables.x[c.0])); + constraints.push(constraint!(chosen == x)); } } constraints } -/// (B) $y_{u,c,i} \le x_u$, except for an unmaterialized query-time -/// producer ([`is_unmaterialized_query_time`]), whose (F) row implies it. +/// (B) $y_{\kappa,u,c,i} \le x$ of $u$'s execution it reads, except for +/// a producer with one reader per context ([`has_one_reader`]), whose (F) row +/// implies it. fn producer_selection_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, @@ -156,70 +203,51 @@ fn producer_selection_constraints( variables .y .iter() - .filter_map(|(&edge, &y)| { - let producer = inputs.physical.edge(edge).producer_candidate_id; - (!is_unmaterialized_query_time(inputs, producer.0)) - .then(|| constraint!(y <= variables.x[producer.0])) + .filter_map(|(&position, &y)| { + let producer = inputs.selectable.edges()[position].producer; + (!has_one_reader(inputs, producer)).then(|| constraint!(y <= variables.x[producer])) }) .collect() } -/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x_v$, except for -/// an unmaterialized query-time candidate -/// ([`is_unmaterialized_query_time`]), whose (F) row implies it. +/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x$ of $v$'s +/// execution answering $q$, except for one with one reader per context +/// ([`has_one_reader`]), whose (F) row implies it. fn query_coverage_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { let mut constraints = Vec::new(); - for (query, candidates) in inputs.query_outputs { - let covered: Expression = candidates.iter().map(|&v| variables.z[&(*query, v)]).sum(); + for (query, executions) in inputs.selectable.answering() { + let covered: Expression = executions + .iter() + .map(|&execution| variables.z[&(*query, execution)]) + .sum(); constraints.push(constraint!(covered == 1)); } - for (&(_, v), &z) in &variables.z { - if !is_unmaterialized_query_time(inputs, v.0) { - constraints.push(constraint!(z <= variables.x[v.0])); + for (&(_, execution), &z) in &variables.z { + if !has_one_reader(inputs, execution) { + constraints.push(constraint!(z <= variables.x[execution])); } } constraints } -/// Whether candidate `v` is a query-time result that is not materialized -/// ($\ell$ = None), so it has at most one consumer and its (F) row is the -/// equality $\sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v} = x_v$ -/// ([`use_constraints`]). With $y, z \ge 0$ that equality implies (B) -/// $y_{v,c,i} \le x_v$ and (D) $z_{q,v} \le x_v$, so those rows are left -/// out for it. -pub(super) fn is_unmaterialized_query_time(inputs: &SelectionInputs<'_>, v: usize) -> bool { - let properties = inputs.physical.physical_candidates[v].physical_properties; - properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime -} - -/// The cost per second of a plan of `candidates` reading over edges that -/// cost `edge_costs` ($c_{u,c,i}$), without the request overhead every plan -/// pays: the candidates' $w_v$, the edges' costs, and each source's raw -/// data on disk once, at its largest selected scan. -pub(super) fn plan_cost_per_second( - inputs: &SelectionInputs<'_>, - candidates: impl IntoIterator, - edge_costs: impl IntoIterator, -) -> f64 { - let mut raw_data_on_disk: BTreeMap<&Source, f64> = BTreeMap::new(); - let mut cost = 0.0; - for v in candidates { - if let Ok(own) = &inputs.costs.candidate_cost_per_second[v] { - cost += own; - } - if let Some((source, on_disk)) = &inputs.costs.raw_data_on_disk[v] { - let kept = raw_data_on_disk.entry(source).or_default(); - *kept = kept.max(*on_disk); - } - } - cost + edge_costs.into_iter().sum::() + raw_data_on_disk.values().sum::() +/// Whether the execution at position `index` has one reader per context +/// (`PhysicalProperties::has_one_reader_per_context`): its (F) row is the equality +/// $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$ ([`use_constraints`]), which +/// with $y, z \ge 0$ implies (B) and (D) for it, so those rows are left +/// out. +pub(super) fn has_one_reader(inputs: &SelectionInputs<'_>, index: usize) -> bool { + let v = inputs.selectable.executions()[index].candidate; + inputs.physical.physical_candidates[v.0] + .physical_properties + .has_one_reader_per_context() } -/// (E) $t_u + 1 \le t_c + n(1 - y_{u,c,i})$, only on edges inside a strongly -/// connected component of the admissible edges, $n$ its size. +/// (E) $t_u + 1 \le t_c + n(1 - y_{\kappa,u,c,i})$ between the producer's +/// and the consumer's executions, only on edges inside a strongly connected +/// component of the selectable edges, $n$ its size. fn acyclicity_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, @@ -228,9 +256,9 @@ fn acyclicity_constraints( variables .y .iter() - .filter_map(|(&edge, &y)| { - let edge = inputs.physical.edge(edge); - let (u, c) = (edge.producer_candidate_id.0, edge.consumer_candidate_id.0); + .filter_map(|(&position, &y)| { + let edge = inputs.selectable.edges()[position]; + let (c, u) = (edge.consumer, edge.producer); if components.of[u] != components.of[c] { return None; } @@ -241,15 +269,14 @@ fn acyclicity_constraints( .collect() } -/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$ for every -/// candidate; with it, a query-time result that is not materialized -/// ($\ell$ = None) has at most one consumer, $\sum_{(v,c,i)} y_{v,c,i} + -/// \sum_q z_{q,v} \le x_v$, so for such a candidate the two are one row, -/// $\sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v} = x_v$. Consumers that share a -/// result read a `OneEvaluation` or a materialized producer (section 5, per -/// execution context). An ingestion-time producer is exempt from the -/// one-consumer rule: there is no `OneEvaluation` at ingestion time, and the -/// consumers of one ingestion run share its rows. +/// (F) $x_{\kappa,v} \le \sum y + \sum_q z_{q,v}$ over the edges and query +/// outputs reading that execution; with it, a candidate with one reader +/// per context (`PhysicalProperties::has_one_reader_per_context`: a query-time result that +/// is not materialized) has $\sum y + \sum_q z_{q,v} \le x_{\kappa,v}$, so +/// for such an execution the two are one row, $\sum y + \sum_q z_{q,v} = +/// x_{\kappa,v}$. An ingestion-time producer is exempt from the +/// one-consumer rule: there is no `OneEvaluation` at ingestion time, and +/// the consumers of one ingestion run share its rows. fn use_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, @@ -257,69 +284,82 @@ fn use_constraints( uses_of(inputs, variables) .into_iter() .enumerate() - .map(|(v, used)| match is_unmaterialized_query_time(inputs, v) { - true => constraint!(used == variables.x[v]), - false => constraint!(variables.x[v] <= used), + .map(|(index, used)| match has_one_reader(inputs, index) { + true => constraint!(used == variables.x[index]), + false => constraint!(variables.x[index] <= used), }) .collect() } /// The raw data of each source is kept on disk once, for the selected scan -/// that keeps the most: $r_s \ge \mathrm{scale} \cdot d_v x_v$ for each -/// query-time `Scan` $v$ of source $s$, $d_v$ its cost per second of the -/// raw data it keeps. The objective's scale is in the row, so $r_s$ is the -/// scaled cost and its objective coefficient is 1. -fn raw_data_disk_constraints( +/// that keeps the most: $r_s \ge \pi_v h_v$ for each query-time `Scan` $v$ +/// of source $s$ with $h_v$, and $r_s \ge d_v x_{\kappa,v}$ for each kept +/// execution of one without, $d_v$ its cost per second of the raw data it +/// keeps retaining everything: each row as ($r_s$, its coefficient, the +/// variable it multiplies), unscaled. +fn raw_data_disk_rows( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, - objective_scale: f64, -) -> Vec { - inputs - .costs - .raw_data_on_disk - .iter() - .enumerate() - .filter_map(|(v, on_disk)| { - let (source, cost) = on_disk.as_ref()?; - Some(constraint!( - variables.raw_data_on_disk[source] >= *cost * objective_scale * variables.x[v] - )) - }) - .collect() + kept: &[bool], +) -> Vec<(Variable, f64, Variable)> { + let costs = inputs.costs; + let mut scans: Vec<(&PhysicalCandidateId, &Source)> = costs.raw_data_source.iter().collect(); + scans.sort_unstable_by_key(|(v, _)| **v); + let mut executions_of: HashMap> = HashMap::new(); + for (index, execution) in inputs.selectable.executions().iter().enumerate() { + if kept[index] { + executions_of + .entry(execution.candidate) + .or_default() + .push(index); + } + } + let mut rows = Vec::new(); + for (v, source) in scans { + let r = variables.raw_data_on_disk[source]; + if let Some(&h) = variables.h.get(v) { + rows.push((r, costs.retention_cost_per_retained_second[v], h)); + } else if let Some(&d) = costs.unbounded_raw_data_on_disk_cost_per_second.get(v) { + for &index in executions_of.get(v).into_iter().flatten() { + rows.push((r, d, variables.x[index])); + } + } + } + rows } -/// Per candidate, its selected edges to consumers and query outputs: -/// $\sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$. +/// Per execution, its selected edges to consumers and query outputs: +/// $\sum y + \sum_q z_{q,v}$. fn uses_of(inputs: &SelectionInputs<'_>, variables: &SelectionVariables) -> Vec { - let mut uses: Vec = (0..inputs.physical.physical_candidates.len()) - .map(|v| { - inputs - .physical - .admissible_edges_out_of(PhysicalCandidateId(v)) - .iter() - .filter_map(|edge| variables.y.get(edge)) - .sum() - }) - .collect(); - for (&(_, v), &z) in &variables.z { - uses[v.0] += z; + let mut uses: Vec = vec![Expression::default(); variables.x.len()]; + for (&position, &y) in &variables.y { + uses[inputs.selectable.edges()[position].producer] += y; + } + for (&(_, execution), &z) in &variables.z { + uses[execution] += z; } uses } -/// The baseline MILP's optimum: $\{x = 1\}$, $\{y = 1\}$ and $\{z = 1\}$, -/// and the MILP's size and time. +/// The MILP's optimum: the executions with $x = 1$, the edges with +/// $y = 1$, the execution answering each query output ($z = 1$) and the +/// retention of each selected candidate with $h_v$, and the MILP's size +/// and time. #[derive(Debug, Clone)] pub(crate) struct SolvedSelection { - /// The selected candidates, ascending. - pub(crate) candidates: Vec, + /// The selected executions, ascending. + pub(crate) executions: Vec, /// The selected edges, ascending. - pub(crate) edges: Vec, - /// The candidate answering each of `SelectionInputs::query_outputs`, in - /// its order. - pub(crate) outputs: Vec, - /// The optimum, unscaled: the selected plan's cost per second. - pub(crate) objective_cost_per_second: f64, + pub(crate) edges: Vec, + /// The execution answering each of `SelectableExecutions::answering`, + /// in its order. + pub(crate) outputs: Vec, + /// $h_v$ in ms of each selected candidate with $h_v$: the least (R) + /// allows for the selection. + pub(crate) retention_ms: HashMap, + /// The optimum, unscaled: the selected plan's cost per second, less + /// the request overhead every plan pays. + pub(crate) cost_per_second_without_request_overhead: f64, /// The MILP's variables and constraint rows. pub(crate) variable_count: usize, pub(crate) constraint_count: usize, @@ -327,48 +367,82 @@ pub(crate) struct SolvedSelection { pub(crate) solve_ms: u64, } -/// Why the baseline MILP has no optimum. +/// A selected $y_{\kappa,u,c,i}$: edge `edge` from execution `producer` +/// into execution `consumer`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SelectedEdge { + pub(crate) consumer: Execution, + pub(crate) edge: PhysicalEdgeId, + pub(crate) producer: Execution, +} + +/// Why the MILP has no optimum. #[derive(Debug, Clone, PartialEq)] pub(crate) enum SelectionMilpError { - /// No plan meets constraints (A)–(F). + /// No plan meets the constraints. Infeasible, /// The solver failed. Solver(String), } -/// Builds and solves the baseline MILP: minimize -/// $\sum_v w_v x_v + \sum_{(u,c,i)} c_{u,c,i} y_{u,c,i} + \sum_s r_s$ under -/// (A)–(F), the one-consumer rule ([`use_constraints`]) and -/// [`raw_data_disk_constraints`], with HiGHS at gap 0 on one thread. The -/// candidates dearer than a feasible plan are left out first -/// ([`dearer_than_a_feasible_plan`]), then the coefficients are scaled so -/// the largest left is 1,000 (HiGHS's tolerances are absolute). +/// Builds and solves the MILP: minimize +/// $\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} +/// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} +/// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a +/// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], +/// [`raw_data_disk_rows`], (R) and (X), with HiGHS at gap 0 on one +/// thread. The executions dearer than a feasible plan +/// ([`dearer_than_a_feasible_plan`]) or dominated by their `OneEvaluation` +/// twin ([`dominated_by_one_evaluation_twin`]) are left out first (and a +/// copy of an equivalent context shares its original's variables, +/// [`copies_in_equivalent_contexts`]), with +/// the $h_v$ of a candidate whose +/// executions are all left out, then the coefficients kept are scaled so +/// the largest is 1,000 (HiGHS's tolerances are absolute). pub(crate) fn solve_selection( inputs: &SelectionInputs<'_>, ) -> Result { let started = Instant::now(); - let left_out = dearer_than_a_feasible_plan(inputs); + let costs = inputs.costs; + let kept: Vec = dearer_than_a_feasible_plan(inputs) + .iter() + .zip(dominated_by_one_evaluation_twin(inputs)) + .zip(&costs.per_execution) + .map(|((&dearer, dominated), cost)| cost.is_ok() && !dearer && !dominated) + .collect(); let components = Components::of(inputs); let mut problem = ProblemVariables::new(); - let variables = selection_variables(inputs, &left_out, &components, &mut problem); - // Scaled by the largest coefficient of a candidate the bound keeps. - let kept = |v: &usize| !left_out[*v]; - let largest = (0..left_out.len()) - .filter(kept) - .filter_map(|v| inputs.costs.candidate_cost_per_second[v].as_ref().ok()) + let variables = selection_variables(inputs, &kept, &components, &mut problem); + // The objective's terms per second, unscaled: (cost, variable). A + // query-time scan's retention is in its source's r_s. + let terms: Vec<(f64, Variable)> = costs + .per_execution + .iter() + .zip(&variables.x) + .zip(&kept) + .filter(|(_, &kept)| kept) + .filter_map(|((cost, &x), _)| Some((*cost.as_ref().ok()?, x))) .chain( variables .y - .keys() - .map(|edge| &inputs.costs.edge_cost_change_per_second[edge]), + .iter() + .map(|(&position, &y)| (costs.per_edge[position].expect("a costed edge"), y)), ) .chain( - (0..left_out.len()) - .filter(kept) - .filter_map(|v| inputs.costs.raw_data_on_disk[v].as_ref()) - .map(|(_, cost)| cost), + variables + .h + .iter() + .filter(|(v, _)| !costs.raw_data_source.contains_key(v)) + .map(|(v, &h)| (costs.retention_cost_per_retained_second[v], h)), ) - .fold(0.0_f64, |largest, &cost| largest.max(cost.abs())); + .collect(); + // Scaled by the largest coefficient kept, the raw-data rows' included. + let raw_data_rows = raw_data_disk_rows(inputs, &variables, &kept); + let largest = terms + .iter() + .map(|(cost, _)| *cost) + .chain(raw_data_rows.iter().map(|&(_, coefficient, _)| coefficient)) + .fold(0.0_f64, |largest, cost| largest.max(cost.abs())); let scale = match largest > 0.0 && largest.is_finite() { true => 1_000.0 / largest, false => 1.0, @@ -379,27 +453,34 @@ pub(crate) fn solve_selection( query_coverage_constraints(inputs, &variables), acyclicity_constraints(inputs, &variables, &components), use_constraints(inputs, &variables), - raw_data_disk_constraints(inputs, &variables, scale), + // The objective's scale is in the raw-data rows, so $r_s$ is the + // scaled cost and its objective coefficient is 1. + raw_data_rows + .iter() + .map(|&(r, coefficient, variable)| constraint!(r >= coefficient * scale * variable)) + .collect(), + retention_constraints( + inputs, + &variables.h, + &variables.x, + &variables.y, + &variables.z, + ), + exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), ] .into_iter() .flatten() .collect(); - let candidate_terms = (0..left_out.len()).filter(kept).filter_map(|v| { - let cost = inputs.costs.candidate_cost_per_second[v].as_ref().ok()?; - Some(cost * scale * variables.x[v]) - }); - let edge_terms = variables - .y + // The raw data on disk is scaled by its rows ([`raw_data_disk_rows`]). + let objective: Expression = terms .iter() - .map(|(edge, &y)| inputs.costs.edge_cost_change_per_second[edge] * scale * y); - // Already scaled by its rows ([`raw_data_disk_constraints`]). - let raw_data_terms = variables - .raw_data_on_disk - .values() - .map(|&on_disk| Expression::from(on_disk)); - let objective: Expression = candidate_terms - .chain(edge_terms) - .chain(raw_data_terms) + .map(|&(cost, v)| cost * scale * v) + .chain( + variables + .raw_data_on_disk + .values() + .map(|&r| Expression::from(r)), + ) .sum(); let variable_count = problem.len(); // HiGHS's columns are the variables in the order they were added. @@ -441,39 +522,51 @@ pub(crate) fn solve_selection( other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), }; let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; - // The optimum, unscaled, with the request overhead every plan pays. - let objective_cost_per_second = plan_cost_per_second( - inputs, - (0..variables.x.len()).filter(|&v| chosen(variables.x[v])), - variables - .y - .iter() - .filter(|&(_, &y)| chosen(y)) - .map(|(edge, _)| inputs.costs.edge_cost_change_per_second[edge]), - ) + inputs.costs.request_overhead_cost_per_second; + let selectable = inputs.selectable; + let executions = selectable.executions(); + let selected: Vec = (0..variables.x.len()) + .filter(|&index| chosen(variables.x[index])) + .collect(); + let selected_edges: Vec = variables + .y + .iter() + .filter(|&(_, &y)| chosen(y)) + .map(|(&position, _)| position) + .collect(); + let answering: Vec<(QueryId, usize)> = selectable + .answering() + .iter() + .map(|(query, answering)| { + let execution = answering + .iter() + .copied() + .find(|&execution| chosen(variables.z[&(*query, execution)])) + .expect("(C): one output per query output reference"); + (*query, execution) + }) + .collect(); + let (cost_per_second_without_request_overhead, retention_ms) = + selection_cost_per_second(inputs, &selected, &selected_edges, &answering); Ok(SolvedSelection { - objective_cost_per_second, - candidates: (0..variables.x.len()) - .filter(|&v| chosen(variables.x[v])) - .map(PhysicalCandidateId) - .collect(), - edges: variables - .y + executions: selected.iter().map(|&index| executions[index]).collect(), + edges: selected_edges .iter() - .filter(|&(_, &y)| chosen(y)) - .map(|(&edge, _)| edge) + .map(|&position| { + let edge = selectable.edges()[position]; + let (consumer, producer) = (edge.consumer, edge.producer); + SelectedEdge { + consumer: executions[consumer], + edge: edge.edge, + producer: executions[producer], + } + }) .collect(), - outputs: inputs - .query_outputs + outputs: answering .iter() - .map(|(query, candidates)| { - candidates - .iter() - .copied() - .find(|&v| chosen(variables.z[&(*query, v)])) - .expect("(C): one output per query output reference") - }) + .map(|&(_, execution)| executions[execution]) .collect(), + retention_ms, + cost_per_second_without_request_overhead, variable_count, constraint_count, construction_ms, @@ -481,5 +574,55 @@ pub(crate) fn solve_selection( }) } +/// The cost per second of a selection, less the request overhead every +/// plan pays: its executions (`selected`, by position in +/// `SelectableExecutions::executions`) and edges (`edges`, by position in +/// `SelectableExecutions::edges`), the retention they and the query +/// outputs `answering` (by execution position) need +/// ([`least_retention_ms`]), and each source's raw data on disk at its +/// largest selected scan; with that retention. +pub(crate) fn selection_cost_per_second( + inputs: &SelectionInputs<'_>, + selected: &[usize], + edges: &[usize], + answering: &[(QueryId, usize)], +) -> (f64, HashMap) { + let costs = inputs.costs; + let retention_ms = least_retention_ms(inputs, edges.iter().copied(), answering.iter().copied()); + let mut raw_data_on_disk: BTreeMap<&Source, f64> = BTreeMap::new(); + let mut retention_cost_per_second = 0.0; + for (&v, &ms) in &retention_ms { + let cost = costs.retention_cost_per_second(v, ms); + match costs.raw_data_source.get(&v) { + Some(source) => { + let kept = raw_data_on_disk.entry(source).or_default(); + *kept = kept.max(cost); + } + None => retention_cost_per_second += cost, + } + } + for &index in selected { + let v = inputs.selectable.executions()[index].candidate; + if let (Some(source), Some(&cost)) = ( + costs.raw_data_source.get(&v), + costs.unbounded_raw_data_on_disk_cost_per_second.get(&v), + ) { + let kept = raw_data_on_disk.entry(source).or_default(); + *kept = kept.max(cost); + } + } + let cost = selected + .iter() + .filter_map(|&index| costs.per_execution[index].as_ref().ok()) + .sum::() + + edges + .iter() + .filter_map(|&position| costs.per_edge[position]) + .sum::() + + retention_cost_per_second + + raw_data_on_disk.values().sum::(); + (cost, retention_ms) +} + #[cfg(test)] mod tests; diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 56c0bb7c7..f5b68ae7b 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -1,29 +1,41 @@ -//! The baseline MILP over small hand-built candidate graphs with given -//! costs: which candidates and edges its optimum selects, and the plan -//! built from it. +//! The selection MILP over small hand-built candidate graphs with given +//! costs: which executions and edges its optimum selects, its retention, +//! and the plan built from it. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{Operator, QueryRoot}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, }; use asap_types::stages::{Lifetime, PhysicalProperties, StorageMedium}; +use asap_types::summary_type::{ExactKind, ExactParams, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_builders::{ - group, own_candidate, query, scan_group, try_candidate, + group, own_candidate, quantile_over_time_readouts, query, scan_group, try_candidate, }; use asap_types::test_support::memo_fixtures; use asap_types::workload::DataWorkload; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, + TimestampMs, +}; use super::*; -use crate::cost::SelectionCosts; +use crate::cost::ExecutionCosts; use crate::plan::plan_of; +use crate::retention::{edge_retention_need_ms, least_retention_ms_per_execution}; +use crate::selectable::{QueryId, SelectableEdge}; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); +const ACROSS_EVALUATIONS: Lifetime = Lifetime::AcrossEvaluations(StorageMedium::Memory); fn scan(memo: &mut Memo) -> MemoGroupId { scan_group(memo, "m", &["job"], vec![]) @@ -104,130 +116,206 @@ fn edge(physical: &PhysicalASAPMemoCandidates, u: usize, c: usize) -> PhysicalEd .0 } -/// Costs `w` per candidate and `edge_costs` per (producer, consumer), 0 for -/// every other edge. -fn costs( - physical: &PhysicalASAPMemoCandidates, - w: &[f64], - edge_costs: &[((usize, usize), f64)], -) -> SelectionCosts { - let mut edge_cost: HashMap = - physical.edges().map(|(id, _)| (id, 0.0)).collect(); - for &((u, c), cost) in edge_costs { - edge_cost.insert(edge(physical, u, c), cost); - } - SelectionCosts { - candidate_cost_per_second: w.iter().map(|&w| Ok(w)).collect(), - edge_cost_change_per_second: edge_cost, - raw_data_on_disk: vec![None; w.len()], - request_overhead_cost_per_second: 0.0, +/// Every query on a 10 s grid, `count` of them. +fn every_10s(count: usize) -> Vec { + (0..count).map(|_| every(10_000)).collect() +} + +/// A query every `interval_ms` at phase 0. +fn every(interval_ms: u32) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("q".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::default(), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, } } -/// Each query output reference of `physical`'s memo with every candidate of -/// its group. -fn outputs(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, Vec)> { - physical - .logical_candidates - .memo - .queries() +/// What the MILP is built over for `physical` and `entries`, with every +/// execution of candidate `v` costing `w[v]`, every edge from `u` to `c` +/// `edge_costs[(u, c)]` (0 when missing) and candidate `v` retaining at +/// `retention[v]` per second retained; each query output answered, as +/// `crate::select_plan` does, by the executions its query may read (X). +fn selection_inputs( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + w: &[f64], + edge_costs: &[((usize, usize), f64)], + retention: &[(usize, f64)], +) -> (SelectableExecutions, ExecutionCosts) { + let mut selectable = SelectableExecutions::new(physical, entries, &output_groups(physical)); + selectable.retain_answering(|query, _, execution| { + !physical.logical_candidates.query_must_not_read( + query.0, + physical.physical_candidates[execution.candidate.0].logical_candidate_id, + ) + }); + let retention_cost_per_retained_second: HashMap = retention .iter() - .enumerate() - .map(|(query, memo_query)| { - let QueryRoot::Operator(group) = memo_query.root else { - unreachable!("operator roots only") - }; - let candidates = (0..physical.physical_candidates.len()) - .filter(|&v| physical.physical_candidates[v].logical_candidate_id.group == group) - .map(PhysicalCandidateId) - .collect(); - (QueryId(query), candidates) - }) - .collect() + .map(|&(v, per_retained_second)| (PhysicalCandidateId(v), per_retained_second)) + .collect(); + let edge_of = |edge: &SelectableEdge| physical.edge(edge.edge); + let mut costs = ExecutionCosts { + per_execution: selectable + .executions() + .iter() + .map(|execution| Ok(w[execution.candidate.0])) + .collect(), + per_edge: selectable + .edges() + .iter() + .map(|selectable_edge| { + let edge = edge_of(selectable_edge); + let key = (edge.producer_candidate_id.0, edge.consumer_candidate_id.0); + Some( + edge_costs + .iter() + .find(|(pair, _)| *pair == key) + .map_or(0.0, |(_, cost)| *cost), + ) + }) + .collect(), + retention_need_ms_per_edge: selectable + .edges() + .iter() + .map(|selectable_edge| { + let edge = edge_of(selectable_edge); + retention_cost_per_retained_second + .contains_key(&edge.producer_candidate_id) + .then(|| edge_retention_need_ms(physical, entries, selectable_edge.edge)) + .flatten() + }) + .collect(), + retention_cost_per_retained_second, + newest_output_retention_ms: HashMap::new(), + least_retention_ms_per_execution: Vec::new(), + raw_data_source: HashMap::new(), + unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), + }; + costs.least_retention_ms_per_execution = + least_retention_ms_per_execution(&selectable, &costs, entries); + (selectable, costs) } +/// Solves over [`selection_inputs`]; returns the solution and the +/// executions it is over. fn solve( physical: &PhysicalASAPMemoCandidates, - costs: &SelectionCosts, -) -> Result { - solve_selection(&SelectionInputs { + entries: &[QueryWorkloadEntry], + w: &[f64], + edge_costs: &[((usize, usize), f64)], + retention: &[(usize, f64)], +) -> ( + Result, + SelectableExecutions, +) { + let (selectable, costs) = selection_inputs(physical, entries, w, edge_costs, retention); + let solved = solve_selection(&SelectionInputs { physical, - costs, - query_outputs: &outputs(physical), - }) + entries, + selectable: &selectable, + costs: &costs, + }); + (solved, selectable) } -fn ids(ids: &[usize]) -> Vec { - ids.iter().copied().map(PhysicalCandidateId).collect() +/// Each query output reference of `physical`'s memo (`Memo::output_groups`). +fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { + physical + .logical_candidates + .memo + .output_groups() + .into_iter() + .map(|(query, group)| (QueryId(query), group)) + .collect() +} + +/// The candidates of `solved`'s executions, one per execution, ascending. +fn candidates(solved: &SolvedSelection) -> Vec { + let mut candidates: Vec = solved + .executions + .iter() + .map(|execution| execution.candidate.0) + .collect(); + candidates.sort_unstable(); + candidates } /// `m[5m]`, its scan with two physical candidates. Through v0 the read /// costs 1 + 5, through v1 2 + 0: the optimum takes v1, and the plan built /// from it is the scan v1 below the range v2, the scan retaining the 5 -/// minutes the range reads. +/// minutes the range reads (R). #[test] fn selects_the_producer_whose_own_and_edge_costs_are_lowest() { let mut memo = empty_memo(); let m = scan(&mut memo); let last_5m = range(&mut memo, 300, m); query(&mut memo, last_5m, None); - let first = own_candidate; let physical = physical( memo, &[ - (first(m), ONE_EVALUATION), - (first(m), NONE), - (first(last_5m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(m), NONE), + (own_candidate(last_5m), NONE), ], ); - let costs = costs(&physical, &[1.0, 2.0, 1.0], &[((0, 2), 5.0)]); - let solved = solve(&physical, &costs).unwrap(); - assert_eq!(solved.candidates, ids(&[1, 2])); - assert_eq!(solved.edges, vec![edge(&physical, 1, 2)]); - assert_eq!(solved.outputs, ids(&[2])); - - let outputs = vec![(QueryId(0), last_5m)]; - let plan = plan_of(&physical, &[], &solved, &outputs).unwrap(); + let (solved, _) = solve( + &physical, + &every_10s(1), + &[1.0, 2.0, 1.0], + &[((0, 2), 5.0)], + &[(1, 1e-6)], + ); + let solved = solved.unwrap(); + assert_eq!(candidates(&solved), vec![1, 2]); + assert_eq!(solved.edges.len(), 1); + assert_eq!(solved.edges[0].edge, edge(&physical, 1, 2)); + assert_eq!(solved.outputs[0].candidate, PhysicalCandidateId(2)); + + let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); assert_eq!(plan.roots, vec![1]); assert_eq!(plan.edges.len(), 1); assert_eq!((plan.edges[0].producer, plan.edges[0].consumer), (0, 1)); - let candidates: Vec<_> = plan + let selected: Vec<_> = plan .nodes .iter() .map(|node| node.selected_physical_candidate) .collect(); assert_eq!( - candidates, + selected, vec![Some(PhysicalCandidateId(1)), Some(PhysicalCandidateId(2))] ); assert_eq!(plan.nodes[0].retention_ms, Some(300_000)); assert_eq!(plan.nodes[1].retention_ms, None); } -/// `m[5m]` and `m[1m]` both read the scan. Its candidate that is not -/// materialized (cost 1) may have one consumer, so the optimum reads the -/// scan's `OneEvaluation` candidate (cost 10) twice rather than both. +/// `m[5m]` and `m[1m]` both read the scan, both queries every 10 s: one +/// context. Its candidate that is not materialized (cost 1) may have one +/// consumer there, so the optimum reads the scan's `OneEvaluation` +/// candidate (cost 10) twice rather than both; without it no plan exists. #[test] -fn a_query_time_result_that_is_not_materialized_has_one_consumer() { +fn a_query_time_result_that_is_not_materialized_has_one_consumer_per_context() { let mut memo = empty_memo(); let m = scan(&mut memo); let last_5m = range(&mut memo, 300, m); let last_1m = range(&mut memo, 60, m); query(&mut memo, last_5m, None); query(&mut memo, last_1m, None); - let first = own_candidate; let physical = physical( memo, &[ - (first(m), NONE), - (first(last_5m), NONE), - (first(last_1m), NONE), - (first(m), ONE_EVALUATION), + (own_candidate(m), NONE), + (own_candidate(last_5m), NONE), + (own_candidate(last_1m), NONE), + (own_candidate(m), ONE_EVALUATION), ], ); - let with_one_evaluation = costs(&physical, &[1.0, 1.0, 1.0, 10.0], &[]); - let solved = solve(&physical, &with_one_evaluation).unwrap(); - assert_eq!(solved.candidates, ids(&[1, 2, 3])); + let (solved, _) = solve(&physical, &every_10s(2), &[1.0, 1.0, 1.0, 10.0], &[], &[]); + assert_eq!(candidates(&solved.unwrap()), vec![1, 2, 3]); let without_one_evaluation = PhysicalASAPMemoCandidates::new( physical.logical_candidates.clone(), @@ -239,11 +327,174 @@ fn a_query_time_result_that_is_not_materialized_has_one_consumer() { .collect(), vec![], ); - let only_unshared = costs(&without_one_evaluation, &[0.0, 1.0, 1.0], &[]); + let (solved, _) = solve( + &without_one_evaluation, + &every_10s(2), + &[0.0, 1.0, 1.0], + &[], + &[], + ); + assert_eq!(solved.unwrap_err(), SelectionMilpError::Infeasible); +} + +/// The same two queries every 10 s and every 15 s: two contexts, each +/// running the scan that is not materialized once for its own query +/// (cost 1 each), cheaper than one `OneEvaluation` run (cost 10), which +/// could not be shared across contexts anyway. +#[test] +fn a_result_that_is_not_materialized_runs_once_per_context_that_reads_it() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + let last_1m = range(&mut memo, 60, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_1m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(last_5m), NONE), + (own_candidate(last_1m), NONE), + (own_candidate(m), ONE_EVALUATION), + ], + ); + let (solved, contexts) = solve( + &physical, + &[every(10_000), every(15_000)], + &[1.0, 1.0, 1.0, 10.0], + &[], + &[], + ); + let solved = solved.unwrap(); + assert_eq!(candidates(&solved), vec![0, 0, 1, 2]); + let scan_contexts: Vec<_> = solved + .executions + .iter() + .filter(|execution| execution.candidate.0 == 0) + .map(|execution| execution.context) + .collect(); + assert_ne!(scan_contexts[0], scan_contexts[1]); + let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); + assert_eq!(plan.nodes.len(), 4); + assert_eq!(contexts.executions().len(), 6); +} + +/// `m[1m]` every 10 s and `m[5m]` every 20 s, both over a chain of 10 s +/// windows of the scan, materialized across evaluations: the chain runs +/// once, in its own context, and delivers on both grids; it retains the +/// longest range its selected consumers read, 5 minutes (R). +#[test] +fn a_materialized_chain_runs_once_and_retains_what_its_consumers_read_back() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let chain = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: m, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 10_000, + }), + ); + let last_1m = range(&mut memo, 60, chain); + let last_5m = range(&mut memo, 300, chain); + query(&mut memo, last_1m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(chain), ACROSS_EVALUATIONS), + (own_candidate(last_1m), NONE), + (own_candidate(last_5m), NONE), + ], + ); + let (solved, _) = solve( + &physical, + &[every(10_000), every(20_000)], + &[1.0, 1.0, 1.0, 1.0], + &[], + &[(1, 1e-3)], + ); + let solved = solved.unwrap(); + assert_eq!(candidates(&solved), vec![0, 1, 2, 3]); + assert_eq!( + solved.retention_ms.get(&PhysicalCandidateId(1)), + Some(&300_000) + ); + let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); + let chain_node = plan + .nodes + .iter() + .find(|node| node.selected_physical_candidate == Some(PhysicalCandidateId(1))) + .unwrap(); + assert_eq!(chain_node.retention_ms, Some(300_000)); +} + +/// A chain of 10 s windows materialized across evaluations, read every +/// 10 s at its newest output only (no range above it): no read needs a +/// retention, but a store always keeps the newest output, so the chain +/// retains one window, 10 s, and pays for it (1 + 1 + 1e-3 · 10 = 2.01), +/// as `plan_cost` charges one retained output. Without the newest output +/// it would retain 0 and pay 2. +#[test] +fn a_stored_result_read_at_its_newest_output_retains_one_output() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let chain = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: m, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 10_000, + }), + ); + query(&mut memo, chain, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(chain), ACROSS_EVALUATIONS), + ], + ); + let entries = every_10s(1); + let solved_with = |newest_output_ms: Option| { + let (selectable, mut costs) = + selection_inputs(&physical, &entries, &[1.0, 1.0], &[], &[(1, 1e-3)]); + if let Some(newest_output_ms) = newest_output_ms { + costs + .newest_output_retention_ms + .insert(PhysicalCandidateId(1), newest_output_ms); + costs.least_retention_ms_per_execution = + least_retention_ms_per_execution(&selectable, &costs, &entries); + } + solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap() + }; + let solved = solved_with(Some(10_000)); assert_eq!( - solve(&without_one_evaluation, &only_unshared).unwrap_err(), - SelectionMilpError::Infeasible + solved.retention_ms.get(&PhysicalCandidateId(1)), + Some(&10_000) ); + assert!((solved.cost_per_second_without_request_overhead - 2.01).abs() < 1e-9); + let without_newest_output = solved_with(None); + assert_eq!( + without_newest_output + .retention_ms + .get(&PhysicalCandidateId(1)), + Some(&0) + ); + assert!((without_newest_output.cost_per_second_without_request_overhead - 2.0).abs() < 1e-9); } /// Ga = `m offset 1s` and Gb = `m offset 2s`, each also computed from the @@ -261,27 +512,31 @@ fn acyclicity_forbids_two_groups_reading_each_other() { let b_from_a = try_candidate(&mut memo, b, shift(1_000, a), None).unwrap(); query(&mut memo, a, None); query(&mut memo, b, None); - let first = own_candidate; // v0 scan; v1 Ga①, v2 Gb①, v3 Ga②, v4 Gb②. let physical = physical( memo, &[ - (first(m), ONE_EVALUATION), - (first(a), ONE_EVALUATION), - (first(b), ONE_EVALUATION), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(a), ONE_EVALUATION), + (own_candidate(b), ONE_EVALUATION), (a_from_b, ONE_EVALUATION), (b_from_a, ONE_EVALUATION), ], ); - let costs = costs(&physical, &[0.0, 10.0, 10.0, 1.0, 1.0], &[]); - let solved = solve(&physical, &costs).unwrap(); + let (solved, _) = solve( + &physical, + &every_10s(2), + &[0.0, 10.0, 10.0, 1.0, 1.0], + &[], + &[], + ); + let solved = solved.unwrap(); + let selected = candidates(&solved); assert!( - solved.candidates == ids(&[0, 1, 4]) || solved.candidates == ids(&[0, 2, 3]), - "{:?}", - solved.candidates + selected == vec![0, 1, 4] || selected == vec![0, 2, 3], + "{selected:?}" ); - let outputs = vec![(QueryId(0), a), (QueryId(1), b)]; - plan_of(&physical, &[], &solved, &outputs).unwrap(); + plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); } /// x08's shape: `max by (job)(count_over_time(m[1m]))`. Path A (scan and @@ -330,7 +585,616 @@ fn a_candidate_dearer_than_a_feasible_plan_does_not_hide_a_cheaper_plan() { 1.666_666_666_666_666_7e-5, 2_553_517.822_748_626_6, ]; - let costs = costs(&physical, &w, &[]); - let solved = solve(&physical, &costs).unwrap(); - assert_eq!(solved.candidates, ids(&[3, 4, 5, 6, 7, 8])); + let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); + assert_eq!(candidates(&solved.unwrap()), vec![3, 4, 5, 6, 7, 8]); + // The unused candidate also retains rows at 2.55e6/s per second + // retained (z05's unused UnivMon chain): it has no h_v once its + // executions are left out, so its retention cost does not scale the + // objective either. + let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[(9, w[9])]); + assert_eq!(candidates(&solved.unwrap()), vec![3, 4, 5, 6, 7, 8]); +} + +/// `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s and Q1 every +/// 30 s, two contexts: its KLL readout (cost 1 per execution, through the +/// state group) is cheaper than the exact quantile (cost 100). Q1 may read +/// its exact-only groups only exactly (a relative target, which the KLL +/// readout's rank error does not meet). With Pass 2's list (the quantile +/// and every group below it) Q1's output is the exact quantile, constraint +/// (X) at the output; with only the state group listed it is too, as the +/// readout would read the state for Q1 (X along the edge); with no list Q1 +/// reads the readout. Q0, with no list, reads the readout throughout. +#[test] +fn a_query_reads_its_exact_only_groups_through_exact_candidates_only() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + let logical_candidates = every_candidate_one_theta_of(&memo, state); + let exact_quantile = logical_candidates + .iter() + .position(|&(candidate, _)| candidate == own_candidate(quantile.result)) + .unwrap(); + let costs: Vec = (0..logical_candidates.len()) + .map(|v| if v == exact_quantile { 100.0 } else { 1.0 }) + .collect(); + let mut physical = physical(memo, &logical_candidates); + // Every group: the quantile and every group below it. + let below_quantile: BTreeSet = + (0..physical.logical_candidates.memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .collect(); + for (q1_exact_only, q1_reads_exactly) in [ + (below_quantile, true), + (BTreeSet::from([state]), true), + (BTreeSet::new(), false), + ] { + physical.logical_candidates.exact_only_groups_of_query = + vec![BTreeSet::new(), q1_exact_only]; + let (solved, _) = solve(&physical, &[every(10_000), every(30_000)], &costs, &[], &[]); + let outputs: Vec = solved + .unwrap() + .outputs + .iter() + .map(|execution| execution.candidate.0) + .collect(); + assert_ne!(outputs[0], exact_quantile); + assert_eq!( + outputs[1] == exact_quantile, + q1_reads_exactly, + "{outputs:?}" + ); + } +} + +/// Every candidate of `memo`'s groups, `OneEvaluation`, with one θ of +/// `state`'s (its first candidate only). +fn every_candidate_one_theta_of( + memo: &Memo, + state: MemoGroupId, +) -> Vec<(CandidateForMemoGroupId, Lifetime)> { + (0..memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .flat_map(|group| { + let count = match group == state { + true => 1, + false => memo.groups()[group.0 as usize].candidates.len(), + }; + (0..count).map(move |index_in_group| { + ( + CandidateForMemoGroupId { + group, + index_in_group, + }, + ONE_EVALUATION, + ) + }) + }) + .collect() +} + +/// The shape of [`a_query_reads_its_exact_only_groups_through_exact_candidates_only`] +/// with Q0 every 10 s, which may read every group, and Q1 every 20 s, +/// which must not read the KLL state; the exact quantile's candidate costs +/// `exact_quantile_cost`, every other 1, and every cost in the 20 s +/// context is half the 10 s one's. The quantile's own candidate is also offered +/// `AcrossEvaluations` at `materialized_cost` (stored on its group's 10 s +/// grid, so it delivers on both; its inputs deliver on no `Materialized` +/// context, so it is never selected), listed first: each query's first +/// answering execution runs in `Materialized(v)`, not in the query's +/// evaluation context. Returns the candidates, the queries, the selection's +/// inputs and the exact quantile's `OneEvaluation` candidate. +fn queries_differing_in_exact_only_groups_with_a_materialized_answer( + materialized_cost: f64, + exact_quantile_cost: f64, +) -> ( + PhysicalASAPMemoCandidates, + [QueryWorkloadEntry; 2], + SelectableExecutions, + ExecutionCosts, + usize, +) { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + let mut candidates = vec![(own_candidate(quantile.result), ACROSS_EVALUATIONS)]; + candidates.extend(every_candidate_one_theta_of(&memo, state)); + let exact_quantile = candidates + .iter() + .position(|&candidate| candidate == (own_candidate(quantile.result), ONE_EVALUATION)) + .unwrap(); + let w: Vec = (0..candidates.len()) + .map(|v| match v { + 0 => materialized_cost, + _ if v == exact_quantile => exact_quantile_cost, + _ => 1.0, + }) + .collect(); + let mut physical = physical(memo, &candidates); + physical.logical_candidates.evaluation_grid_of_group[quantile.result.0 as usize] = + Some(asap_types::stages::EvaluationGrid { + period_ms: 10_000, + phase_ms: 0, + }); + physical.logical_candidates.exact_only_groups_of_query = + vec![BTreeSet::new(), BTreeSet::from([state])]; + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let every_20s = |index: usize| { + matches!( + selectable.context(selectable.executions()[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 20_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_20s(index) { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + for (position, edge) in selectable.edges().iter().enumerate() { + if every_20s(edge.consumer) { + costs.per_edge[position] = costs.per_edge[position].map(|cost| cost / 2.0); + } + } + (physical, entries, selectable, costs, exact_quantile) +} + +/// [`queries_differing_in_exact_only_groups_with_a_materialized_answer`]: +/// the two evaluation contexts have the same executions at half the cost, +/// but their queries read different groups, so they are not copies +/// ([`copies_in_equivalent_contexts`]): each query is in its own +/// evaluation context (`SelectableExecutions::evaluation_context`), not +/// in that of its first answering execution, `Materialized(v)`. +#[test] +fn contexts_whose_queries_read_different_groups_are_not_copies() { + let (physical, entries, selectable, costs, _) = + queries_differing_in_exact_only_groups_with_a_materialized_answer(1.0, 100.0); + let first_answering = selectable.executions()[selectable.answering()[0].1[0]]; + assert!(matches!( + selectable.context(first_answering.context), + asap_types::stages::ExecutionContext::Materialized(_) + )); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let shared = copies_in_equivalent_contexts(&inputs, &vec![true; selectable.executions().len()]); + assert!(shared + .original_of_execution + .iter() + .enumerate() + .all(|(index, &original)| original == index)); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(1)]); +} + +/// [`queries_differing_in_exact_only_groups_with_a_materialized_answer`] +/// with the exact quantile costing 4 (2 in the 20 s context), every other +/// candidate 1: the optimum reads the KLL readout for Q0 (its five +/// candidates, 5) and the exact quantile for Q1 (0.5 + 0.5 + 2), 8. +/// Sharing the two contexts' variables would make Q0 read the exact +/// quantile too, at 9. +#[test] +fn queries_reading_different_groups_select_differently_in_contexts_alike() { + let (physical, entries, selectable, costs, exact_quantile) = + queries_differing_in_exact_only_groups_with_a_materialized_answer(1.0, 4.0); + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + let outputs: Vec = solved + .outputs + .iter() + .map(|execution| execution.candidate.0) + .collect(); + assert_ne!(outputs[0], exact_quantile, "{outputs:?}"); + assert_eq!(outputs[1], exact_quantile, "{outputs:?}"); + assert!( + (solved.cost_per_second_without_request_overhead - 8.0).abs() < 1e-9, + "{}", + solved.cost_per_second_without_request_overhead + ); +} + +/// x07's shape: `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s, +/// whose only paths read the KLL state (its exact quantile cannot be +/// costed), and by Q1 every 30 s, which may read every group only exactly. +/// The greedy bound leaves out of each query's trees only what that query +/// must not read, so it finds a plan (Q0 through the state, Q1 through the +/// exact quantile) and leaves out an unused scan costing 1e6/s; leaving +/// out of every query's trees what some query must not read found none. +#[test] +fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + let mut candidates = every_candidate_one_theta_of(&memo, state); + let exact_quantile = candidates + .iter() + .position(|&(candidate, _)| candidate == own_candidate(quantile.result)) + .unwrap(); + let unused_scan = candidates.len(); + candidates.push((own_candidate(m), NONE)); + let w: Vec = (0..candidates.len()) + .map(|v| match v { + _ if v == unused_scan => 1e6, + _ if v == exact_quantile => 100.0, + _ => 1.0, + }) + .collect(); + let mut physical = physical(memo, &candidates); + physical.logical_candidates.exact_only_groups_of_query = vec![ + BTreeSet::new(), + (0..physical.logical_candidates.memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .collect(), + ]; + let entries = [every(10_000), every(30_000)]; + let (mut selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let q0_context = selectable.executions()[selectable.answering()[0].1[0]].context; + for (index, execution) in selectable.executions().iter().enumerate() { + if execution.candidate.0 == exact_quantile && execution.context == q0_context { + costs.per_execution[index] = Err("not costable".into()); + } + } + selectable.retain_answering(|_, index, _| costs.per_execution[index].is_ok()); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let left_out = dearer_than_a_feasible_plan(&inputs); + let unused: Vec = selectable + .executions() + .iter() + .zip(&left_out) + .filter(|(execution, _)| execution.candidate.0 == unused_scan) + .map(|(_, &left_out)| left_out) + .collect(); + assert_eq!(unused, vec![true, true]); + assert!(solve_selection(&inputs).is_ok()); +} + +/// `m[5m]` every 10 s over a scan that is not materialized (cost 1) or a +/// `OneEvaluation` one (cost 2, retaining at 1e6/s per second retained). +/// The greedy plan reads the first, at 2/s. The second's own cost alone is +/// no dearer than that, but (R) makes it retain the 5 minutes its one +/// consumer reads, 3e8/s: the bound leaves it out. +#[test] +fn the_bound_counts_the_least_retention_an_execution_pays() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = every_10s(1); + let (selectable, costs) = + selection_inputs(&physical, &entries, &[1.0, 2.0, 1.0], &[], &[(1, 1e6)]); + let left_out = dearer_than_a_feasible_plan(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }); + let left_out: Vec = selectable + .executions() + .iter() + .zip(&left_out) + .filter(|(_, &left_out)| left_out) + .map(|(execution, _)| execution.candidate.0) + .collect(); + assert_eq!(left_out, vec![1]); +} + +/// `m[5m]` every 10 s over a scan offered with ℓ = None (v0) and +/// `OneEvaluation` (v1). At equal cost and with the same consumer, v1 +/// dominates v0, which is left out; v0 is kept when v1 costs more, or when +/// v1 cannot be read by v0's consumer (its edge is not costed). +#[test] +fn a_none_execution_its_one_evaluation_twin_dominates_is_left_out() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = every_10s(1); + let dominated = |w: &[f64], twin_edge_costed: bool| { + let (selectable, mut costs) = selection_inputs(&physical, &entries, w, &[], &[]); + for (position, &SelectableEdge { edge, .. }) in selectable.edges().iter().enumerate() { + if physical.edge(edge).producer_candidate_id.0 == 1 && !twin_edge_costed { + costs.per_edge[position] = None; + } + } + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let candidates: Vec = selectable + .executions() + .iter() + .zip(dominated_by_one_evaluation_twin(&inputs)) + .filter(|(_, dominated)| *dominated) + .map(|(execution, _)| execution.candidate.0) + .collect(); + let solved = solve_selection(&inputs).unwrap(); + (candidates, self::candidates(&solved)) + }; + assert_eq!(dominated(&[1.0, 1.0, 1.0], true), (vec![0], vec![1, 2])); + assert_eq!(dominated(&[1.0, 2.0, 1.0], true), (vec![], vec![0, 2])); + assert_eq!(dominated(&[1.0, 1.0, 1.0], false), (vec![], vec![0, 2])); +} + +/// The same query `m[5m]` every 10 s and every 20 s: two evaluation +/// contexts with the same executions and edges, each cost in the 20 s one +/// half the 10 s one's (half the runs). The 20 s context is a copy of the +/// 10 s one ([`copies_in_equivalent_contexts`]): its executions, edges and +/// query share their variables, and the optimum reads the scan's +/// `OneEvaluation` candidate in both (1.5 + 0 against 1 + 2), costing +/// 2.5 + 1.25. With one cost of the 20 s context not half, they are not +/// copies; nor are they when the queries' accuracy targets differ (rank +/// 0.01 every 10 s, rank 0.05 every 20 s), which the accuracy constraint +/// (K) reads per query. +#[test] +fn a_context_equivalent_to_another_shares_its_variables() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = + selection_inputs(&physical, &entries, &[1.0, 1.5, 1.0], &[((0, 2), 2.0)], &[]); + let executions = selectable.executions(); + let every_20s = |index: usize| { + matches!( + selectable.context(executions[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 20_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_20s(index) { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + for (position, edge) in selectable.edges().iter().enumerate() { + if every_20s(edge.consumer) { + costs.per_edge[position] = costs.per_edge[position].map(|cost| cost / 2.0); + } + } + let kept = vec![true; executions.len()]; + let copies = |costs: &ExecutionCosts| { + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs, + }; + copies_in_equivalent_contexts(&inputs, &kept) + }; + let SharedVariables { + original_of_execution, + original_of_edge, + original_of_query, + } = copies(&costs); + for (index, &original) in original_of_execution.iter().enumerate() { + match every_20s(index) { + true => { + assert!(!every_20s(original)); + assert_eq!(executions[original].candidate, executions[index].candidate); + } + false => assert_eq!(original, index), + } + } + for (position, &original) in original_of_edge.iter().enumerate() { + let edge = selectable.edges()[position]; + assert_eq!(original != position, every_20s(edge.consumer)); + assert_eq!(selectable.edges()[original].edge, edge.edge); + } + assert_eq!(original_of_query, vec![QueryId(0), QueryId(0)]); + + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + assert_eq!(candidates(&solved), vec![1, 1, 2, 2]); + assert!((solved.cost_per_second_without_request_overhead - 3.75).abs() < 1e-9); + assert_eq!(solved.variable_count, 3 + 2 + 1); + + let first_of_20s = (0..executions.len()) + .find(|&index| every_20s(index)) + .unwrap(); + let rank = + |max_error| AccuracyTarget::max_error(max_error).accuracy_metric(AccuracyMetric::Rank); + let mut differing_targets = entries.clone(); + differing_targets[0].requirements.accuracy = rank(0.01); + differing_targets[1].requirements.accuracy = rank(0.05); + let shared = copies_in_equivalent_contexts( + &SelectionInputs { + physical: &physical, + entries: &differing_targets, + selectable: &selectable, + costs: &costs, + }, + &kept, + ); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(1)]); + assert!(shared + .original_of_execution + .iter() + .enumerate() + .all(|(index, &original)| original == index)); + + costs.per_execution[first_of_20s] = Ok(10.0); + assert!(copies(&costs) + .original_of_execution + .iter() + .enumerate() + .all(|(index, &original)| original == index)); +} + +/// The same query `m[5m]` every 60 s and every 300 s, each cost in the +/// 300 s context a fifth of the 60 s one's, except the scan-to-range +/// edge: a difference of the range's cost through the scan and its own, +/// as RollupHeavy-8users' flows component computes it, +/// 2.811002367077395e-9 − 2.8110047226529567e-9 (−2.3556e-15) in the 60 s +/// context and 5.622004734154791e-10 − 5.622009445305915e-10 (−4.7112e-16) +/// in the 300 s one. The two differences are in the ratio 0.200000000026, +/// off by more than 1e-12 of themselves but not of the contexts' costs, +/// so the contexts are copies. +#[test] +fn contexts_whose_small_edge_costs_differ_by_rounding_are_copies() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = [every(60_000), every(300_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &[1.0, 1.5, 1.0], &[], &[]); + let executions = selectable.executions(); + let every_300s = |index: usize| { + matches!( + selectable.context(executions[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 300_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_300s(index) { + *cost = Ok(cost.clone().unwrap() / 5.0); + } + } + let scan_to_range = edge(&physical, 0, 2); + for (position, selectable_edge) in selectable.edges().iter().enumerate() { + if selectable_edge.edge == scan_to_range { + costs.per_edge[position] = Some(match every_300s(selectable_edge.consumer) { + true => 5.622004734154791e-10 - 5.622009445305915e-10, + false => 2.811002367077395e-9 - 2.8110047226529567e-9, + }); + } + } + let shared = copies_in_equivalent_contexts( + &SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }, + &vec![true; executions.len()], + ); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(0)]); + for (index, &original) in shared.original_of_execution.iter().enumerate() { + assert_eq!(original != index, every_300s(index)); + } +} + +/// The same query `m[5m]` every 60 s and every 300 s over a scan with +/// ℓ = None (s0) or `OneEvaluation` (s1), read by the range (r) or by a +/// `OneEvaluation` range costing 1e12 (2e11 every 300 s), which the bound +/// leaves out. Every 60 s, r, s0 and s1 cost 1, 1 and 1.1; every 300 s +/// 0.2, 0.3 and 0.22, and the edge s1 → r 1e-3 (so s1 does not dominate +/// s0). The 300 s costs are not one multiple of the 60 s ones (s0 would +/// be 0.2), so the contexts are not copies, and the optimum reads s0 +/// every 60 s and s1 every 300 s: 2 + 0.421 = 2.421. Compared by every +/// cost, the left-out range's 2e11 sets the tolerance to 0.2, the +/// contexts share their variables, and the optimum is 2.5. +#[test] +fn a_left_out_execution_does_not_set_the_tolerance_of_context_copies() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + (own_candidate(last_5m), ONE_EVALUATION), + ], + ); + let entries = [every(60_000), every(300_000)]; + let (selectable, mut costs) = + selection_inputs(&physical, &entries, &[1.0, 1.1, 1.0, 1e12], &[], &[]); + let executions = selectable.executions(); + let every_300s = |index: usize| { + matches!( + selectable.context(executions[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 300_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_300s(index) { + *cost = Ok([0.3, 0.22, 0.2, 2e11][executions[index].candidate.0]); + } + } + let s1_to_range = edge(&physical, 1, 2); + for (position, selectable_edge) in selectable.edges().iter().enumerate() { + if selectable_edge.edge == s1_to_range && every_300s(selectable_edge.consumer) { + costs.per_edge[position] = Some(1e-3); + } + } + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + assert!( + (solved.cost_per_second_without_request_overhead - 2.421).abs() < 1e-9, + "{}", + solved.cost_per_second_without_request_overhead + ); + assert_eq!(candidates(&solved), vec![0, 1, 2, 2]); } diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs new file mode 100644 index 000000000..efc40d8b8 --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -0,0 +1,133 @@ +//! The retention extension of the selection MILP +//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +//! section 5): the retention amount $h_v$ and constraint (R). The +//! executions $x_{\kappa,v}$ and edges $y_{\kappa,u,c,i}$ it selects over +//! are [`crate::selectable::SelectableExecutions`]'s, in [`super::model`]. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; + +use asap_types::stages::PhysicalCandidateId; + +use super::model::SelectionInputs; +use crate::retention::ms_to_seconds; +use crate::selectable::QueryId; +use crate::selectable::Reader; + +/// $h_v$ in seconds retained (the unit keeps (R)'s coefficients near the +/// others'), added to `problem`, for each candidate with a retention cost +/// ([`crate::cost::ExecutionCosts::retention_cost_per_retained_second`]) +/// one of whose executions is `kept` (by position in +/// `SelectableExecutions::executions`): one whose executions are all left +/// out retains nothing. +pub(crate) fn retention_variables( + inputs: &SelectionInputs<'_>, + kept: &[bool], + problem: &mut ProblemVariables, +) -> BTreeMap { + let retaining = &inputs.costs.retention_cost_per_retained_second; + let mut retained: Vec = inputs + .selectable + .executions() + .iter() + .zip(kept) + .filter(|(execution, &kept)| kept && retaining.contains_key(&execution.candidate)) + .map(|(execution, _)| execution.candidate) + .collect(); + retained.sort_unstable(); + retained.dedup(); + retained + .into_iter() + .map(|v| (v, problem.add(variable().min(0)))) + .collect() +} + +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$ and +/// $h_v \ge \operatorname{span}_q\, z_{q,v}$, for a candidate $v$ with $h_v$: +/// how far back its rows are read through each selectable edge `y` (the +/// reading path's offset plus range, +/// [`crate::cost::ExecutionCosts::retention_need_ms_per_edge`]) and by each +/// query reading it directly (its range-evaluation span), a constant per +/// edge, so (R) is linear. Query latency is ignored: a result counts as +/// read when its evaluation starts. +/// +/// With them, per execution $u$ of such a $v$, $h_v \ge +/// \min_{\text{uses}} \operatorname{need} \cdot x_u$ over its costed +/// reads (`ExecutionCosts::least_retention_ms_per_execution`): (F) gives a +/// selected execution a use, and (R) then retains at least the least +/// need. It is implied for integer $x$, $y$, +/// $z$, but tightens the relaxation where fractional $y$ leave $h_v$ low. +pub(crate) fn retention_constraints( + inputs: &SelectionInputs<'_>, + h: &BTreeMap, + x: &[Variable], + y: &BTreeMap, + z: &BTreeMap<(QueryId, usize), Variable>, +) -> Vec { + let executions = inputs.selectable.executions(); + let mut constraints = Vec::new(); + for (index, reader, need_ms) in inputs + .selectable + .reads_back(&inputs.costs.retention_need_ms_per_edge, inputs.entries) + { + let Some(&h) = h.get(&executions[index].candidate) else { + continue; + }; + let read = match reader { + Reader::Edge(position) => y.get(&position), + Reader::Output(query) => z.get(&(query, index)), + }; + if let Some(&read) = read { + let need_ms = need_ms.expect("a candidate with h_v is read within a bound"); + constraints.push(constraint!(h >= ms_to_seconds(need_ms) * read)); + } + } + for (index, need_ms) in inputs + .costs + .least_retention_ms_per_execution + .iter() + .enumerate() + { + if let (Some(&h), Some(need_ms)) = (h.get(&executions[index].candidate), *need_ms) { + if need_ms > 0 { + constraints.push(constraint!(h >= ms_to_seconds(need_ms) * x[index])); + } + } + } + constraints +} + +/// The least $h_v$ (R) allows for a selection, in ms, of each candidate +/// with a retention cost that `edges` (selected, by position in +/// `SelectableExecutions::edges`) or `outputs` (the query outputs and the +/// executions answering them, by position) read, each read's need at +/// least its newest output (`ExecutionCosts::retention_ms_for_need`): what +/// the MILP's optimum retains when $\pi_v > 0$, and what the plan retains. +pub(crate) fn least_retention_ms( + inputs: &SelectionInputs<'_>, + edges: impl IntoIterator, + outputs: impl IntoIterator, +) -> HashMap { + let executions = inputs.selectable.executions(); + let retaining = &inputs.costs.retention_cost_per_retained_second; + let edges: HashSet = edges.into_iter().collect(); + let outputs: HashSet<(QueryId, usize)> = outputs.into_iter().collect(); + let mut retention_ms: HashMap = HashMap::new(); + for (index, reader, need_ms) in inputs + .selectable + .reads_back(&inputs.costs.retention_need_ms_per_edge, inputs.entries) + { + let selected = match reader { + Reader::Edge(position) => edges.contains(&position), + Reader::Output(query) => outputs.contains(&(query, index)), + }; + let v = executions[index].candidate; + if let (true, Some(need_ms), true) = (selected, need_ms, retaining.contains_key(&v)) { + let need_ms = inputs.costs.retention_ms_for_need(v, need_ms); + let most = retention_ms.entry(v).or_default(); + *most = (*most).max(need_ms); + } + } + retention_ms +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index d1c641bd6..92da7901e 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -1,6 +1,6 @@ -//! An upper bound on the baseline MILP's optimum, and the candidates it +//! An upper bound on the selection MILP's optimum, and the executions it //! rules out: the cost of one feasible plan found greedily, and every -//! candidate that alone costs more than it. Leaving those out before the +//! execution that alone costs more than it. Leaving those out before the //! objective is scaled keeps a candidate costing millions per second from //! shrinking the differences between plans costing thousandths under the //! solver's tolerances. @@ -12,160 +12,173 @@ //! greedy plan bounds the optimum by about 0.0088/s, so the chain is left //! out and the largest coefficient left is about 0.006/s. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; -use asap_types::stages::PhysicalCandidateId; +use super::model::{has_one_reader, selection_cost_per_second, SelectionInputs}; +use super::COST_RELATIVE_TOLERANCE; +use crate::selectable::QueryId; -use super::model::{is_unmaterialized_query_time, plan_cost_per_second, SelectionInputs}; - -/// How many times the greedy plan is rebuilt without a candidate that +/// How many times the greedy plan is rebuilt without an execution that /// made it infeasible before giving up on a bound. const REPAIRS: usize = 32; -/// Per (consumer, input), the costed edges into it, each as (producer, -/// $c_{u,c,i}$). -type EdgesInto = HashMap<(usize, usize), Vec<(usize, f64)>>; - -/// Per input of a candidate, the edge its cheapest tree reads it over, as -/// (producer, $c_{u,c,i}$). -type TreeInputs = Vec>; +/// Per input of an execution, the edge its cheapest tree reads it over, as +/// (edge position, producer execution). +type TreeInputs = Vec>; -/// Per candidate, whether it costs more than a feasible plan -/// ([`feasible_plan_cost_per_second`]): its own $w_v$ plus the most its -/// input edges can take off ($\min(0, c_{u,v,i})$ per input) is over the -/// bound. Such a candidate is in no optimal plan, as long as every other -/// selected candidate costs at least 0 with its selected inputs. None is -/// ruled out when no feasible plan is found. +/// Per execution, by position in `SelectableExecutions::executions`, +/// whether it costs more than a feasible plan +/// ([`feasible_plan_cost_per_second`]): its own cost, plus the least +/// retention $\pi_v h_v$ (R) makes it pay, plus the most its input edges +/// can take off ($\min(0, c_{\kappa,u,v,i})$ per input) is over the +/// bound. Such an execution is in no optimal plan, as long as every +/// other selected execution costs at least 0 with its selected inputs. +/// None is ruled out when no feasible plan is found. pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { - let count = inputs.physical.physical_candidates.len(); + let selectable = inputs.selectable; + let count = selectable.executions().len(); let Some(bound) = feasible_plan_cost_per_second(inputs) else { return vec![false; count]; }; - let mut cheapest_input_edge: HashMap<(PhysicalCandidateId, usize), f64> = HashMap::new(); - for (&edge, &cost) in &inputs.costs.edge_cost_change_per_second { - let edge = inputs.physical.edge(edge); - let key = (edge.consumer_candidate_id, edge.input_index_of_consumer); - let cheapest = cheapest_input_edge.entry(key).or_insert(0.0); - *cheapest = cheapest.min(cost); - } + let costs = inputs.costs; + let executions = selectable.executions(); + // The least retention (R) makes each execution pay; a query-time + // scan's retention is its source's $r_s$, left out. + let least_retention_cost = |index: usize| { + let v = executions[index].candidate; + match costs.least_retention_ms_per_execution[index] { + Some(need_ms) if !costs.raw_data_source.contains_key(&v) => { + costs.retention_cost_per_second(v, need_ms) + } + _ => 0.0, + } + }; + // Looser than `COST_RELATIVE_TOLERANCE` on purpose: the bound and each + // execution's least cost are sums over many executions, and leaving + // out an execution the optimum needs cannot be undone, so an execution + // is left out only when it is clearly dearer (by 1e-9 of the bound; + // 1e-21/s when the bound is 0). let tolerance = 1e-9 * bound.abs().max(1e-12); (0..count) - .map(PhysicalCandidateId) - .map(|v| { - let Ok(own) = &inputs.costs.candidate_cost_per_second[v.0] else { + .map(|index| { + let Ok(own) = &costs.per_execution[index] else { return false; }; let least = own - + (0..inputs.physical.input_count(v)) - .filter_map(|i| cheapest_input_edge.get(&(v, i))) + + least_retention_cost(index) + + (0..selectable.input_count(index)) + .map(|i| { + selectable + .edges_into(index, i) + .iter() + .filter_map(|&position| costs.per_edge[position]) + .fold(0.0, f64::min) + }) .sum::(); least > bound + tolerance }) .collect() } -/// The cost per second (without the request overhead every plan pays) of -/// a feasible plan: each query output reads the cheapest tree of -/// candidates below it, each candidate's input from the producer whose -/// tree is cheapest ($w_v + \sum_i \min_u (c_{u,v,i} + \text{tree}_u)$). -/// Trees share the candidates they have in common; when that gives a -/// query-time result that is not materialized two consumers, or a cycle, -/// the plan is rebuilt without that candidate. `None` when no feasible -/// plan is found that way. +/// The cost per second (less the request overhead every plan pays) of a +/// feasible plan ([`selection_cost_per_second`]): each query output reads +/// the cheapest tree of executions below it, each execution's input from +/// the producer whose tree is cheapest ($\rho_\kappa c_{\kappa,v} + \sum_i +/// \min_u (\rho_\kappa c_{\kappa,u,v,i} + \text{tree}_u)$), over the +/// executions its query may read (constraint (X)). The query outputs are +/// added in order, a later tree reusing the executions an earlier one +/// added. When that gives an execution with one reader per context a +/// second consumer, closes a cycle, or reuses a tree with an execution +/// the query must not read, the plan is rebuilt with that execution left +/// out of the query's trees. `None` when no feasible plan is found that +/// way. fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { - let physical = inputs.physical; - let count = physical.physical_candidates.len(); - // Each (consumer, input)'s costed edges: (edge, producer, c_{u,c,i}). - let mut into: EdgesInto = HashMap::new(); - for (&id, &cost) in &inputs.costs.edge_cost_change_per_second { - let edge = physical.edge(id); - into.entry((edge.consumer_candidate_id.0, edge.input_index_of_consumer)) - .or_default() - .push((edge.producer_candidate_id.0, cost)); - } - let inputs_of: Vec = (0..count) - .map(|v| physical.input_count(PhysicalCandidateId(v))) + let selectable = inputs.selectable; + let executions = selectable.executions(); + let count = executions.len(); + let one_reader: Vec = (0..count) + .map(|index| has_one_reader(inputs, index)) .collect(); - let mut excluded = vec![false; count]; - for _ in 0..REPAIRS { - let (tree_cost, chosen) = cheapest_trees(inputs, &into, &inputs_of, &excluded); - // Each query output's cheapest candidate, then the trees below. - let mut uses = vec![0usize; count]; - let mut outputs = Vec::new(); - for (_, candidates) in inputs.query_outputs { - let best = candidates + let forbidden = |query: QueryId, e: usize| { + inputs.physical.logical_candidates.query_must_not_read( + query.0, + inputs.physical.physical_candidates[executions[e].candidate.0].logical_candidate_id, + ) + }; + // Per query, the executions its trees leave out: those it must not + // read, and those a repair left out for it. + let mut excluded: BTreeMap> = BTreeMap::new(); + for (query, _) in selectable.answering() { + excluded + .entry(*query) + .or_insert_with(|| (0..count).map(|e| forbidden(*query, e)).collect()); + } + // The cheapest trees, per set of executions left out. + let mut trees: HashMap, (Vec, Vec)> = HashMap::new(); + 'repair: for _ in 0..REPAIRS { + let mut plan = GreedyPlan::new(count); + let mut answering: Vec<(QueryId, usize)> = Vec::new(); + for (query, outputs) in selectable.answering() { + let left_out = &excluded[query]; + let (tree_cost, chosen) = trees + .entry(left_out.clone()) + .or_insert_with(|| cheapest_trees(inputs, left_out)); + let best = outputs .iter() - .map(|v| v.0) - .filter(|&v| tree_cost[v].is_finite()) + .copied() + .filter(|&index| tree_cost[index].is_finite()) .min_by(|&a, &b| tree_cost[a].total_cmp(&tree_cost[b]))?; - uses[best] += 1; - outputs.push(best); - } - let plan = match plan_below(&outputs, &chosen, &inputs_of) { - Ok(plan) => plan, - Err(on_cycle) => { - excluded[on_cycle] = true; - continue; + answering.push((*query, best)); + if let Err(repair) = plan.add_tree(best, chosen, &one_reader, |e| forbidden(*query, e)) + { + excluded.get_mut(query).expect("a query")[repair] = true; + continue 'repair; } - }; - for &(v, _) in &plan.edges { - uses[v] += 1; - } - if let Some(shared) = plan - .candidates - .iter() - .copied() - .find(|&v| is_unmaterialized_query_time(inputs, v) && uses[v] > 1) - { - excluded[shared] = true; - continue; } - let cost = plan_cost_per_second( - inputs, - plan.candidates.iter().copied(), - plan.edges.iter().map(|&(_, cost)| cost), - ); + let (cost, _) = + selection_cost_per_second(inputs, &plan.executions, &plan.edges, &answering); return cost.is_finite().then_some(cost); } None } -/// Per candidate not `excluded`, the cost of its cheapest tree and, per +/// Per execution not `excluded`, the cost of its cheapest tree and, per /// input, the edge it reads it over: relaxed until no tree gets cheaper -/// (at most once per candidate), so a tree over a cycle of candidates is +/// (at most once per execution), so a tree over a cycle of executions is /// never cheaper than one without it when costs are not negative. -fn cheapest_trees( - inputs: &SelectionInputs<'_>, - into: &EdgesInto, - inputs_of: &[usize], - excluded: &[bool], -) -> (Vec, Vec) { - let count = inputs_of.len(); +fn cheapest_trees(inputs: &SelectionInputs<'_>, excluded: &[bool]) -> (Vec, Vec) { + let selectable = inputs.selectable; + let count = excluded.len(); let mut tree_cost = vec![f64::INFINITY; count]; - let mut chosen: Vec = inputs_of.iter().map(|&n| vec![None; n]).collect(); + let mut chosen: Vec = (0..count) + .map(|v| vec![None; selectable.input_count(v)]) + .collect(); for _ in 0..count.max(1) { let mut changed = false; for v in 0..count { - let Ok(own) = &inputs.costs.candidate_cost_per_second[v] else { + let Ok(own) = &inputs.costs.per_execution[v] else { continue; }; if excluded[v] { continue; } let mut cost = *own; - let mut reads = Vec::with_capacity(inputs_of[v]); - for i in 0..inputs_of[v] { - let cheapest = into - .get(&(v, i)) - .into_iter() - .flatten() - .filter(|&&(u, _)| !excluded[u] && tree_cost[u].is_finite()) - .map(|&(u, c)| (u, c, c + tree_cost[u])) + let mut reads = Vec::with_capacity(selectable.input_count(v)); + for i in 0..selectable.input_count(v) { + let cheapest = selectable + .edges_into(v, i) + .iter() + .filter_map(|&position| { + let c = inputs.costs.per_edge[position]?; + let u = selectable.edges()[position].producer; + (!excluded[u] && tree_cost[u].is_finite()) + .then(|| (position, u, c + tree_cost[u])) + }) .min_by(|a, b| a.2.total_cmp(&b.2)); match cheapest { - Some((u, c, through)) => { + Some((position, u, through)) => { cost += through; - reads.push(Some((u, c))); + reads.push(Some((position, u))); } None => { cost = f64::INFINITY; @@ -173,7 +186,9 @@ fn cheapest_trees( } } } - if cost < tree_cost[v] - 1e-12 * cost.abs().max(1e-300) { + // Cheaper by more than rounding (the smallest positive + // magnitude when the cost is 0). + if cost < tree_cost[v] - COST_RELATIVE_TOLERANCE * cost.abs().max(1e-300) { tree_cost[v] = cost; chosen[v] = reads; changed = true; @@ -188,49 +203,93 @@ fn cheapest_trees( /// A plan of the trees below some query outputs. struct GreedyPlan { - /// Its candidates, each once. - candidates: Vec, - /// Its edges, as (producer, $c_{u,c,i}$). - edges: Vec<(usize, f64)>, + /// Its executions, each once, producers first. + executions: Vec, + /// Its edges, by position in `SelectableExecutions::edges`. + edges: Vec, + /// Per execution, its producers in the plan; `None` when not in it yet. + producers: Vec>>, } -/// The plan of the trees `chosen` below `outputs`; Err with a candidate -/// on a cycle of them. -fn plan_below( - outputs: &[usize], - chosen: &[TreeInputs], - inputs_of: &[usize], -) -> Result { - // 0 unvisited, 1 on the walk, 2 done. - let mut state = vec![0u8; inputs_of.len()]; - let mut plan = GreedyPlan { - candidates: Vec::new(), - edges: Vec::new(), - }; - for &output in outputs { - if state[output] == 2 { - continue; +impl GreedyPlan { + /// The empty plan over `count` executions. + fn new(count: usize) -> Self { + Self { + executions: Vec::new(), + edges: Vec::new(), + producers: vec![None; count], + } + } + + /// Adds the tree `chosen` below `output` (each execution's inputs, by + /// execution), reusing the executions already in the plan with their + /// trees; Err with an execution to leave out of the query's trees: one + /// with `one_reader` per context read a second time, one closing a + /// cycle, or one already in the plan whose tree reads an execution + /// `forbidden` to the query. + fn add_tree( + &mut self, + output: usize, + chosen: &[TreeInputs], + one_reader: &[bool], + forbidden: impl Fn(usize) -> bool, + ) -> Result<(), usize> { + if self.producers[output].is_some() { + return self.reuse(output, one_reader, &forbidden); } + // Executions on the walk, not yet in the plan. + let mut on_walk = vec![false; self.producers.len()]; + on_walk[output] = true; let mut stack = vec![(output, 0usize)]; - state[output] = 1; + let mut walked: Vec> = vec![Vec::new(); self.producers.len()]; while let Some((v, i)) = stack.pop() { - if i == inputs_of[v] { - state[v] = 2; - plan.candidates.push(v); + if i == chosen[v].len() { + on_walk[v] = false; + self.producers[v] = Some(std::mem::take(&mut walked[v])); + self.executions.push(v); continue; } stack.push((v, i + 1)); - let (u, c) = chosen[v][i].expect("a finite tree reads every input"); - plan.edges.push((u, c)); - match state[u] { - 0 => { - state[u] = 1; + let (position, u) = chosen[v][i].expect("a finite tree reads every input"); + self.edges.push(position); + walked[v].push(u); + if on_walk[u] { + return Err(u); + } + match self.producers[u] { + Some(_) => self.reuse(u, one_reader, &forbidden)?, + None => { + on_walk[u] = true; stack.push((u, 0)); } - 1 => return Err(u), - _ => {} } } + Ok(()) + } + + /// Reads `execution`, already in the plan, a second time: Err with it + /// when it has `one_reader` per context or its tree reads an execution + /// `forbidden` to the reader's query. + fn reuse( + &self, + execution: usize, + one_reader: &[bool], + forbidden: &impl Fn(usize) -> bool, + ) -> Result<(), usize> { + if one_reader[execution] { + return Err(execution); + } + let mut pending = vec![execution]; + let mut seen = vec![false; self.producers.len()]; + while let Some(e) = pending.pop() { + if std::mem::replace(&mut seen[e], true) { + continue; + } + if forbidden(e) { + return Err(execution); + } + pending.extend(self.producers[e].iter().flatten()); + } + Ok(()) } - Ok(plan) } diff --git a/crates/stage3-plan-selection/src/plan.rs b/crates/stage3-plan-selection/src/plan.rs index f955df74e..658a6a408 100644 --- a/crates/stage3-plan-selection/src/plan.rs +++ b/crates/stage3-plan-selection/src/plan.rs @@ -1,8 +1,10 @@ -//! The plan Stage 3 builds from the baseline MILP's optimum as its last -//! step (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +//! The plan Stage 3 builds from the MILP's optimum as its last step +//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 3, "Plan output"): one `PhysicalASAPDAG` node per selected -//! candidate ([`candidate_node`], which costing builds too), its inputs -//! from the selected edges, each node's `selected_physical_candidate` set. +//! execution ([`candidate_node`], which costing builds too), its inputs +//! from the selected edges, each node's `selected_physical_candidate` set +//! and `retention_ms` its candidate's $h_v$. Several nodes may come from +//! one candidate, one per context it runs in. use std::collections::{BTreeMap, HashMap}; @@ -12,13 +14,10 @@ use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, }; use asap_types::plan::view_coverage; -use asap_types::stages::{ - Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, -}; -use asap_types::workload::QueryWorkloadEntry; +use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; -use crate::milp::model::{QueryId, SolvedSelection}; -use crate::retention::retention_need_ms; +use crate::milp::model::{SelectedEdge, SolvedSelection}; +use crate::selectable::{Execution, QueryId}; /// Candidate `v` as a plan node with id `id`, its input `i` read from node /// `input_ids[i]`: its operator, timing, lifetime and θ, its group's schema @@ -35,9 +34,8 @@ pub(crate) fn candidate_node( let logical = memo.candidate(candidate.logical_candidate_id); let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize].group_properties; let mut inputs = input_ids.iter(); - let payload = logical - .expression - .operator + let payload = physical + .operator(v) .map_children(|_| *inputs.next().expect("one node per input")); let lifetime = candidate.physical_properties.lifetime(); PhysicalASAPDAGNode { @@ -52,7 +50,9 @@ pub(crate) fn candidate_node( output_schema: group.schema.clone(), retention_ms: None, selected_physical_candidate: Some(v), - view_coverage: matches!(lifetime, Lifetime::AcrossEvaluations(_)) + view_coverage: candidate + .physical_properties + .is_across_evaluations() .then(|| view_coverage(memo, candidate.logical_candidate_id.group)) .flatten(), } @@ -60,97 +60,75 @@ pub(crate) fn candidate_node( /// The plan of `solved`, whose output `j` answers `outputs[j]` (a query and /// the group its root, or one operator its scalar root, reads). Nodes are -/// numbered producers first. A node that retains rows (materialized, or a -/// query-time `Scan`) has `retention_ms` the most its selected consumers -/// read back ([`retention_need_ms`]), and at least the range-evaluation -/// span of a query it answers directly. +/// numbered producers first. A node whose candidate has $h_v$ retains that +/// much (`retention_ms`); one that retains rows a consumer reads unbounded +/// has `retention_ms` `None` and keeps every row. pub(crate) fn plan_of( physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], solved: &SolvedSelection, outputs: &[(QueryId, MemoGroupId)], ) -> Result { - let mut producer_of: HashMap<(PhysicalCandidateId, usize), PhysicalCandidateId> = - HashMap::new(); - let mut consumers_of: BTreeMap> = BTreeMap::new(); - for &id in &solved.edges { - let edge = physical.edge(id); + let mut producer_of: HashMap<(Execution, usize), Execution> = HashMap::new(); + let mut consumers_of: BTreeMap> = BTreeMap::new(); + for &SelectedEdge { + consumer, + edge, + producer, + } in &solved.edges + { producer_of.insert( - (edge.consumer_candidate_id, edge.input_index_of_consumer), - edge.producer_candidate_id, + (consumer, physical.edge(edge).input_index_of_consumer), + producer, ); - consumers_of - .entry(edge.producer_candidate_id) - .or_default() - .push(id); + consumers_of.entry(producer).or_default().push(consumer); } - let consumers = |v: PhysicalCandidateId| consumers_of.get(&v).into_iter().flatten().copied(); - let readers_of = - |v: PhysicalCandidateId| consumers(v).map(|edge| physical.edge(edge).consumer_candidate_id); - let inputs = |v: PhysicalCandidateId| -> Vec { - (0..physical.input_count(v)) - .map(|i| producer_of[&(v, i)]) - .collect() - }; + let input_count = |execution: Execution| physical.input_count(execution.candidate); // Producers first: Kahn's order over the selected edges, which (E) // keeps acyclic. - let mut waiting: HashMap = solved - .candidates + let mut waiting: HashMap = solved + .executions .iter() - .map(|&v| (v, physical.input_count(v))) + .map(|&execution| (execution, input_count(execution))) .collect(); - let mut ready: Vec = solved - .candidates + let mut ready: Vec = solved + .executions .iter() .rev() .copied() - .filter(|v| waiting[v] == 0) + .filter(|execution| waiting[execution] == 0) .collect(); - let mut node_of: HashMap = HashMap::new(); - let mut order = Vec::with_capacity(solved.candidates.len()); - while let Some(v) = ready.pop() { - node_of.insert(v, order.len()); - order.push(v); - for edge in consumers(v) { - let consumer = physical.edge(edge).consumer_candidate_id; - let left = waiting.get_mut(&consumer).expect("a selected consumer"); + let mut node_of: HashMap = HashMap::new(); + let mut order = Vec::with_capacity(solved.executions.len()); + while let Some(execution) = ready.pop() { + node_of.insert(execution, order.len()); + order.push(execution); + for consumer in consumers_of.get(&execution).into_iter().flatten() { + let left = waiting.get_mut(consumer).expect("a selected consumer"); *left -= 1; if *left == 0 { - ready.push(consumer); + ready.push(*consumer); } } } - if order.len() != solved.candidates.len() { + if order.len() != solved.executions.len() { return Err(PhysicalASAPDAGValidationError::Cycle); } - // The span of the queries each candidate answers directly. - let mut output_span_ms: HashMap = HashMap::new(); - for (&(query, _), &v) in outputs.iter().zip(&solved.outputs) { - let span_ms = entries - .get(query.0) - .and_then(|entry| entry.range_evaluation) - .map_or(0, |range| range.span_ms); - let known = output_span_ms.entry(v).or_default(); - *known = (*known).max(span_ms); - } let nodes = order .iter() .enumerate() - .map(|(id, &v)| { - let input_ids: Vec = - inputs(v).iter().map(|producer| node_of[producer]).collect(); - let mut node = candidate_node(physical, v, id, &input_ids); - if node.retains_rows() { - node.retention_ms = retention_need_ms(physical, entries, v, readers_of) - .map(|need_ms| need_ms.max(output_span_ms.get(&v).copied().unwrap_or(0))); - } + .map(|(id, &execution)| { + let input_ids: Vec = (0..input_count(execution)) + .map(|i| node_of[&producer_of[&(execution, i)]]) + .collect(); + let mut node = candidate_node(physical, execution.candidate, id, &input_ids); + node.retention_ms = solved.retention_ms.get(&execution.candidate).copied(); node }) .collect(); let output_node: HashMap<(QueryId, MemoGroupId), PhysicalASAPNodeId> = outputs .iter() .zip(&solved.outputs) - .map(|(&output, v)| (output, node_of[v])) + .map(|(&output, execution)| (output, node_of[execution])) .collect(); let mut roots = Vec::new(); let mut scalar_roots = Vec::new(); diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index 31768b6af..e999389ad 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -13,7 +13,7 @@ use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidate, PrunedCandidate}; use thiserror::Error; -use crate::cost::SelectionCosts; +use crate::cost::ExecutionCosts; /// Peak memory above which a candidate's report carries a warning when the /// deployment declares no memory capacity: 10^12 bytes (1 TB), more than @@ -193,12 +193,12 @@ pub enum ErrorSelectingPlan { UnsupportedCostModel(String), } -/// The selection MILP is infeasible, with the reasons candidates could not +/// The selection MILP is infeasible, with the reasons executions could not /// be costed. -pub(crate) fn infeasible(costs: &SelectionCosts) -> ErrorSelectingPlan { +pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { let mut reasons: Vec<&str> = Vec::new(); for reason in costs - .candidate_cost_per_second + .per_execution .iter() .filter_map(|cost| cost.as_ref().err()) { @@ -207,11 +207,12 @@ pub(crate) fn infeasible(costs: &SelectionCosts) -> ErrorSelectingPlan { } } let mut reason = "no plan gives every query output a candidate whose every input has an \ - admissible producer (the selection MILP is infeasible)" + admissible producer that delivers on its context (the selection MILP is \ + infeasible)" .to_string(); if !reasons.is_empty() { reason += &format!( - "; candidates left out because they could not be costed: {}", + "; executions left out because they could not be costed: {}", reasons.join("; ") ); } @@ -296,35 +297,4 @@ fn sketches_below( } #[cfg(test)] -mod tests { - use super::*; - use asap_cost::{ClusterLimits, ClusterObjective, ResourceUnitCost}; - - /// Without a declared memory capacity nothing rejects a candidate for - /// its memory, so a peak over `LARGE_PEAK_MEMORY_BYTES` is reported - /// as a warning (the eval's 24 h quantiles plan peaked at 1.43 TB). - /// Under cost model 3 the cluster's memory is declared and its capacity - /// check applies instead, so there is no warning. - #[test] - fn a_very_large_peak_memory_is_warned_about_only_without_a_declared_capacity() { - let usage_over_time = CostModel::UsageOverTime(ResourceUnitCost::illustrative()); - assert_eq!( - large_peak_memory_warning(1_430_000_000_000, &usage_over_time).as_deref(), - Some( - "peak memory 1430000000000 bytes is over 1000000000000 bytes and the deployment \ - declares no memory capacity to check it against" - ) - ); - assert_eq!( - large_peak_memory_warning(LARGE_PEAK_MEMORY_BYTES, &usage_over_time), - None - ); - let cluster = CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective: ClusterObjective::LowestLatency, - }); - assert_eq!(large_peak_memory_warning(1_430_000_000_000, &cluster), None); - } -} +mod tests; diff --git a/crates/stage3-plan-selection/src/report/tests.rs b/crates/stage3-plan-selection/src/report/tests.rs new file mode 100644 index 000000000..c73f67f71 --- /dev/null +++ b/crates/stage3-plan-selection/src/report/tests.rs @@ -0,0 +1,30 @@ +use super::*; +use asap_cost::{ClusterLimits, ClusterObjective, ResourceUnitCost}; + +/// Without a declared memory capacity nothing rejects a candidate for +/// its memory, so a peak over `LARGE_PEAK_MEMORY_BYTES` is reported +/// as a warning (the eval's 24 h quantiles plan peaked at 1.43 TB). +/// Under cost model 3 the cluster's memory is declared and its capacity +/// check applies instead, so there is no warning. +#[test] +fn a_very_large_peak_memory_is_warned_about_only_without_a_declared_capacity() { + let usage_over_time = CostModel::UsageOverTime(ResourceUnitCost::illustrative()); + assert_eq!( + large_peak_memory_warning(1_430_000_000_000, &usage_over_time).as_deref(), + Some( + "peak memory 1430000000000 bytes is over 1000000000000 bytes and the deployment \ + declares no memory capacity to check it against" + ) + ); + assert_eq!( + large_peak_memory_warning(LARGE_PEAK_MEMORY_BYTES, &usage_over_time), + None + ); + let cluster = CostModel::ClusterCapacity(ClusterLimits { + memory_bytes: u64::MAX, + disk_bytes: u64::MAX, + cpu_cores: None, + objective: ClusterObjective::LowestLatency, + }); + assert_eq!(large_peak_memory_warning(1_430_000_000_000, &cluster), None); +} diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 0e19a2bd6..15d8d3f3b 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -1,73 +1,119 @@ //! How far back from the evaluation time a candidate that retains rows (a -//! materialized one, or a query-time `Scan`) must keep them, and how far -//! back one execution of a query-time `Scan` reads, for the candidates -//! that may read it: [`asap_cost::read_back_ms`] over candidates +//! materialized one, or a query-time `Scan`) must keep them for a reader +//! over one edge, $\operatorname{need}_{\kappa,u,c,i}$ of constraint (R), +//! or for a query reading it directly, and how far back one execution of a +//! query-time `Scan` reads for that reader: [`asap_cost::read_back_ms`] +//! over candidates; and every read of an execution's rows +//! (`SelectableExecutions::reads_back`) //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). -use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::ir::Operator; use asap_types::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; -use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; +use asap_types::stages::{ + MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, +}; use asap_types::workload::QueryWorkloadEntry; -/// How far back from the evaluation time `producer`'s rows must be kept -/// for its readers (`readers_of`, the candidates that may read it): -/// [`asap_cost::read_back_ms`] with, at each reader, the longest -/// range-evaluation span of the queries reaching it. -pub(crate) fn retention_need_ms>( +use crate::cost::ExecutionCosts; +use crate::selectable::{Reader, SelectableExecutions}; + +/// $\operatorname{need}_{\kappa,u,c,i}$: how far back from the evaluation +/// time `edge`'s producer $u$ must keep its rows for its consumer $c$, with +/// $c$ $u$'s only reader and every candidate that may read $c$ above it +/// ([`read_back`]), at each reader the longest range-evaluation span of +/// the queries reaching it; `None` when read unbounded. +/// +/// Example: `TimeRange(5m)` over a chain of 10 s windows, read by a range +/// query with a 1 h span: 3,900,000 ms. +pub(crate) fn edge_retention_need_ms( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], - producer: PhysicalCandidateId, - readers_of: impl Fn(PhysicalCandidateId) -> R, + edge: PhysicalEdgeId, ) -> Option { let memo = &physical.logical_candidates.memo; - let span_ms = |v: PhysicalCandidateId| { + read_back(physical, edge, |v| { let group = physical.physical_candidates[v.0].logical_candidate_id.group; memo.groups()[group.0 as usize] .reached_by_queries() .iter() - .filter_map(|&query| entries.get(query)?.range_evaluation) - .map(|range| range.span_ms) + .filter_map(|&query| entries.get(query)) + .map(QueryWorkloadEntry::range_span_ms) .max() .unwrap_or(0) - }; - read_back(physical, producer, readers_of, span_ms) + }) } -/// How far back from the evaluation time one execution of the query-time -/// `Scan` `producer` reads, for its readers (`readers_of`): -/// [`asap_cost::read_back_ms`] without a range-evaluation span. -pub(crate) fn scan_read_ms>( +/// How far back one execution of the query-time `Scan` that is `edge`'s +/// producer reads for its consumer ([`read_back`] without a +/// range-evaluation span); `None` when read whole. +pub(crate) fn edge_scan_read_ms( physical: &PhysicalASAPMemoCandidates, - producer: PhysicalCandidateId, - readers_of: impl Fn(PhysicalCandidateId) -> R, + edge: PhysicalEdgeId, ) -> Option { - read_back(physical, producer, readers_of, |_| 0) + read_back(physical, edge, |_| 0) +} + +/// The least retention, in ms, over each execution's costed reads +/// (`SelectableExecutions::reads_back`: an edge out of it `costs.per_edge` +/// costs, or a query output it may answer), each read's need at least its +/// candidate's newest output (`ExecutionCosts::retention_ms_for_need`), by +/// position in [`SelectableExecutions::executions`]; `None` without a +/// bounded one. (F) gives a selected execution a use and (R) then retains +/// at least that: the greedy bound charges it, and the MILP has +/// $h_v \ge$ it $\cdot x$. +pub(crate) fn least_retention_ms_per_execution( + selectable: &SelectableExecutions, + costs: &ExecutionCosts, + entries: &[QueryWorkloadEntry], +) -> Vec> { + let mut least_retention_ms = vec![None; selectable.executions().len()]; + for (index, reader, need_ms) in + selectable.reads_back(&costs.retention_need_ms_per_edge, entries) + { + let costed = match reader { + Reader::Edge(position) => costs.per_edge[position].is_some(), + Reader::Output(_) => true, + }; + if let (true, Some(need_ms)) = (costed, need_ms) { + let retention_ms = + costs.retention_ms_for_need(selectable.executions()[index].candidate, need_ms); + let least: &mut Option = &mut least_retention_ms[index]; + *least = Some(least.map_or(retention_ms, |least| least.min(retention_ms))); + } + } + least_retention_ms +} + +/// `ms` in seconds, the unit of $h_v$ in the MILP and of the costs per +/// second retained. +pub(crate) fn ms_to_seconds(ms: u64) -> f64 { + ms as f64 / 1_000.0 } -/// [`asap_cost::read_back_ms`] over candidates ([`Candidates`]). -fn read_back>( +/// [`asap_cost::read_back_ms`] of `edge`'s producer over candidates, read +/// by `edge`'s consumer only and above it by every candidate that may read +/// one (its admissible edges out), each read as its operator at its timing +/// reads ([`Candidates`]), with `span_ms` at each reader. +fn read_back( physical: &PhysicalASAPMemoCandidates, - producer: PhysicalCandidateId, - readers_of: impl Fn(PhysicalCandidateId) -> R, + edge: PhysicalEdgeId, span_ms: impl Fn(PhysicalCandidateId) -> u64, ) -> Option { - let readers_of = |v| readers_of(v).into_iter().collect(); let candidates = Candidates { physical, - readers_of: &readers_of, + edge: physical.edge(edge), }; - asap_cost::read_back_ms(&candidates, producer, span_ms) + asap_cost::read_back_ms(&candidates, candidates.edge.producer_candidate_id, span_ms) } -/// Stage 3's candidates as the time walks see them: the readers a caller -/// gives (those that may read a candidate, or the selected ones), each -/// candidate's admissible producers as its inputs, its operator at its -/// timing and lifetime ([`InputTimeAccess::of`]), and a current-time read -/// of a `Scan` (raw data) needing every row. +/// Stage 3's candidates as the time walks see them above `edge`: its +/// producer read by its consumer only, any other candidate as +/// `PhysicalASAPMemoCandidates` sees it (`TimeIndexedPlan`: by every +/// candidate that may read it, over its admissible edges out). struct Candidates<'a> { physical: &'a PhysicalASAPMemoCandidates, - readers_of: &'a dyn Fn(PhysicalCandidateId) -> Vec, + edge: &'a PhysicalEdge, } impl TimeIndexedPlan for Candidates<'_> { @@ -75,39 +121,25 @@ impl TimeIndexedPlan for Candidates<'_> { type Child = MemoGroupId; fn readers_of(&self, v: PhysicalCandidateId) -> Vec { - (self.readers_of)(v) + match v == self.edge.producer_candidate_id { + true => vec![self.edge.consumer_candidate_id], + false => self.physical.readers_of(v), + } } fn inputs_of(&self, v: PhysicalCandidateId) -> Vec { - (0..self.physical.input_count(v)) - .flat_map(|input| self.physical.admissible_edges_into(v, input)) - .map(|&edge| self.physical.edge(edge).producer_candidate_id) - .collect() + self.physical.inputs_of(v) } fn input_time_access(&self, v: PhysicalCandidateId) -> InputTimeAccess { - let properties = self.physical.physical_candidates[v.0].physical_properties; - InputTimeAccess::of( - self.operator_of(v), - properties.timing(), - properties.lifetime(), - ) + self.physical.input_time_access(v) } fn current_time_read_needs_every_row(&self, v: PhysicalCandidateId) -> bool { - matches!( - self.operator_of(v), - Operator::NonASAP(NonASAPOp::Scan { .. }) - ) + self.physical.current_time_read_needs_every_row(v) } fn operator_of(&self, v: PhysicalCandidateId) -> &Operator { - &self - .physical - .logical_candidates - .memo - .candidate(self.physical.physical_candidates[v.0].logical_candidate_id) - .expression - .operator + self.physical.operator(v) } } diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index f97f30726..06d399316 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -1,123 +1,496 @@ -//! What the baseline selection may select: the selectable edges (each -//! admissible edge whose producer delivers on every evaluation its -//! consumer serves) and, per query output, the candidates that may answer -//! it. +//! What the selection may select +//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +//! section 5, "Execution contexts, by example"): the distinct execution +//! contexts $\kappa$ of a workload, every execution of a candidate in a +//! context, the edges $y_{\kappa,u,c,i}$ between executions, and per query +//! output the executions that may answer it ([`SelectableExecutions::new`]). +//! +//! - A query is evaluated in the evaluation context of its grid +//! (`QueryWorkloadEntry::evaluation_grid`); queries on one grid share it, a range query's +//! steps included. +//! - A candidate with $\ell$ = None or `OneEvaluation` runs once per context +//! that selects it ($x_{\kappa,v}$), the context of its consumer; at +//! ingestion time, where each arriving row is computed once whatever +//! reads it (`asap_cost::computations`), in the one ingestion context +//! `ExecutionContext::EVERY_ARRIVING_ROW`. +//! - A candidate with $\ell$ = `AcrossEvaluations` runs once, in its own +//! context `Materialized(v)` ($x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$). +//! Its inputs run in that context at query time; at ingestion time its +//! producers must deliver on the ingestion pipeline of its refresh +//! window. +//! - The input of a `TimeShift` anchored with `@ end()` or `@ start()` runs +//! on the grid of the request end (start) times, once per request of a +//! range query, in a context of its own per anchor: a request's start, +//! its end and an evaluation are different times, so their results are +//! not shared. An `@ end()` read by an instant query is anchored at its +//! evaluation time, so its input runs in the evaluation context +//! (`asap_types::stages::anchor_is_evaluation_time`). +//! - An input of a consumer in $\kappa$ reads a producer, over an admissible +//! edge, only where the producer delivers on $\kappa$ +//! (`PhysicalASAPMemoCandidates::delivers_on`). -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; +use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::QueryRoot; +use asap_types::plan::materialized_view::RefreshWindow; use asap_types::stages::{ - output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, - MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, + anchor_is_evaluation_time, producer_satisfies_physical_requirements, queries_of_context, + query_reads, EvaluationGrid, ExecutionContext, ExecutionContextId, MemoGroupId, + PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, }; -use asap_types::workload::QueryWorkloadEntry; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; -use crate::cost::SelectionCosts; -use crate::milp::model::QueryId; +/// A query, by index in the workload. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct QueryId(pub usize); -/// The edges a plan may select: admissible, with a producer that delivers -/// on every evaluation its query-time consumer serves -/// ([`delivers_to_consumer`]). -pub(crate) fn selectable_edges( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], -) -> Vec { - physical - .edges() - .filter(|(_, edge)| { - edge.is_logically_and_physically_admissible() - && delivers_to_consumer(physical, entries, edge) - }) - .map(|(id, _)| id) - .collect() +/// What reads an execution's rows back ([`SelectableExecutions::reads_back`]): a selectable edge +/// out of it, by position in [`SelectableExecutions::edges`], or a query +/// output it may answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Reader { + Edge(usize), + Output(QueryId), } -/// Each query output reference (a query, and the group its operator root, -/// or one operator of its scalar root, reads), in order. -pub(crate) fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { - let roots: Vec> = physical - .logical_candidates - .memo - .queries() - .iter() - .map(|query| query.root.clone()) - .collect(); - output_references(&roots) - .into_iter() - .map(|(query, group)| (QueryId(query), group)) - .collect::>() - .into_iter() - .collect() +/// One run of candidate `candidate` in context `context`: a node of the +/// plan. $x^{\mathrm{mat}}_v$ is the execution of $v$ in +/// `Materialized(v)`, $x_{\kappa,v}$ that of $v$ in $\kappa$. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct Execution { + pub(crate) context: ExecutionContextId, + pub(crate) candidate: PhysicalCandidateId, } -/// Per output of `outputs`, the candidates of its group that may answer -/// it: costed, readable at query time and delivering on its query's -/// evaluations. -pub(crate) fn query_outputs( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - costs: &SelectionCosts, - outputs: &[(QueryId, MemoGroupId)], -) -> Vec<(QueryId, Vec)> { - outputs - .iter() - .map(|&(query, group)| { - let candidates = physical - .physical_candidates - .iter() - .enumerate() - .filter(|(v, candidate)| { - candidate.logical_candidate_id.group == group - && costs.candidate_cost_per_second[*v].is_ok() - && producer_satisfies_physical_requirements( - candidate, - ExecutionTiming::QueryTime, - ) - && candidate.delivers_on(physical, &evaluation_context(&entries[query.0])) - }) - .map(|(v, _)| PhysicalCandidateId(v)) - .collect(); - (query, candidates) - }) - .collect() +/// One $y_{\kappa,u,c,i}$ the selection may choose: edge `edge` from the +/// producer execution `producer` into the consumer execution `consumer`, +/// executions by position in [`SelectableExecutions::executions`]; +/// $\kappa$ is the consumer's context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct SelectableEdge { + pub(crate) consumer: usize, + pub(crate) edge: PhysicalEdgeId, + pub(crate) producer: usize, +} + +/// A context's request anchor ([`SelectableExecutions::request_anchor`]): +/// the anchor of the `TimeShift` whose inputs it runs, and the queries +/// whose requests it runs for. +type RequestAnchor = Option<(AtModifier, Vec)>; + +/// The distinct contexts of a workload and every execution and edge the +/// selection may choose, with the executions that may answer each query +/// output ([`Self::new`]). Executions are referred to by position in +/// [`Self::executions`]. +pub(crate) struct SelectableExecutions { + /// The distinct contexts, by [`ExecutionContextId`]. + contexts: Vec, + /// Per context, by [`ExecutionContextId`]: for the inputs of a + /// `TimeShift` anchored with `@ end()` or `@ start()`, the anchor and + /// the queries whose requests they run for (those on the shift's + /// evaluation grid that reach it), ascending; `None` for any other + /// context. + request_anchor_of_context: Vec, + /// The id of each context with its request anchor. + context_ids: HashMap<(ExecutionContext, RequestAnchor), ExecutionContextId>, + /// Per memo group, each (query, anchor) whose walk over the memo reaches + /// it ([`query_reads`]): the anchor of the `TimeShift` above it it is + /// read per request below, `None` in the query's evaluation context. + reads_of_group: HashMap)>>, + /// Every execution reached from a query output, ascending. + executions: Vec, + /// Every $y_{\kappa,u,c,i}$, ascending. + edges: Vec, + /// Per execution, by position, and per input of its candidate, the + /// positions in [`Self::edges`] of the edges into it. + edges_into: Vec>>, + /// Per execution, by position, the positions in [`Self::edges`] of the + /// edges out of it. + edges_out_of: Vec>, + /// Each query output reference (a query, and the group its operator + /// root, or one operator of its scalar root, reads), in + /// `Memo::output_groups` order, with the executions, by position, that may + /// answer it: $z_{q,v}$ exists only for them. + answering: Vec<(QueryId, Vec)>, + /// Per query, by [`QueryId`], the evaluation context it is evaluated + /// in: `Evaluation { grid: entry.evaluation_grid() }`, no request + /// anchor; `None` for a query with no output reference. Its answering + /// executions may run elsewhere (an `AcrossEvaluations` one in its own + /// `Materialized(v)`), so this is not read off them. + /// + /// Example: a query every 60 s is evaluated in the context of grid + /// 60 s at phase 0. + evaluation_context_of_query: Vec>, } -/// The evaluation context of `entry`'s query: the grid its evaluations -/// fall on (`QueryRecurrence::evaluation_grid_ms`), anchored at the epoch. -/// Times that are not known may fall on any millisecond: a 1 ms grid, on -/// which only producers stored at every time deliver. -fn evaluation_context(entry: &QueryWorkloadEntry) -> ExecutionContext { - ExecutionContext::Evaluation { - grid: EvaluationGrid { - period_ms: entry - .recurrence - .evaluation_grid_ms(entry.range_evaluation) - .unwrap_or(1), - phase_ms: 0, - }, +impl SelectableExecutions { + /// The contexts, executions and edges of `physical` for the queries + /// `entries` (by query index), reached from `outputs` (each query output + /// reference, a query and the group it reads, `Memo::output_groups`), and + /// per output the executions that may answer it: a candidate of its + /// group a query-time consumer may read + /// (`producer_satisfies_physical_requirements`) that delivers on the + /// query's context. + pub(crate) fn new( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + outputs: &[(QueryId, MemoGroupId)], + ) -> Self { + let mut selectable = Self { + reads_of_group: reads_of_groups(physical, entries, outputs), + contexts: Vec::new(), + request_anchor_of_context: Vec::new(), + context_ids: HashMap::new(), + executions: Vec::new(), + edges: Vec::new(), + edges_into: Vec::new(), + edges_out_of: Vec::new(), + answering: Vec::new(), + evaluation_context_of_query: vec![None; entries.len()], + }; + let mut reached: BTreeSet = BTreeSet::new(); + let mut pending: Vec = Vec::new(); + let mut answering = Vec::with_capacity(outputs.len()); + for &(query, group) in outputs { + let context = selectable.context_id( + ExecutionContext::Evaluation { + grid: entries[query.0].evaluation_grid(), + }, + None, + ); + selectable.evaluation_context_of_query[query.0] = Some(context); + let mut executions = Vec::new(); + for (v, candidate) in physical.physical_candidates.iter().enumerate() { + if candidate.logical_candidate_id.group != group + || !producer_satisfies_physical_requirements( + candidate, + ExecutionTiming::QueryTime, + ) + { + continue; + } + if let Some(execution) = + selectable.execution_reading(physical, context, PhysicalCandidateId(v)) + { + executions.push(execution); + if reached.insert(execution) { + pending.push(execution); + } + } + } + answering.push((query, executions)); + } + let mut edges = BTreeSet::new(); + while let Some(consumer) = pending.pop() { + let c = consumer.candidate; + let context = selectable.input_context(physical, entries, consumer); + for input in 0..physical.input_count(c) { + for &edge in physical.admissible_edges_into(c, input) { + let u = physical.edge(edge).producer_candidate_id; + if let Some(producer) = selectable.execution_reading(physical, context, u) { + edges.insert((consumer, edge, producer)); + if reached.insert(producer) { + pending.push(producer); + } + } + } + } + } + selectable.executions = reached.into_iter().collect(); + let position = |execution: Execution| { + selectable + .executions + .binary_search(&execution) + .expect("a reached execution") + }; + selectable.edges = edges + .into_iter() + .map(|(consumer, edge, producer)| SelectableEdge { + consumer: position(consumer), + edge, + producer: position(producer), + }) + .collect(); + selectable.answering = answering + .into_iter() + .map(|(query, executions)| (query, executions.into_iter().map(position).collect())) + .collect(); + selectable.edges_into = selectable + .executions + .iter() + .map(|execution| vec![Vec::new(); physical.input_count(execution.candidate)]) + .collect(); + selectable.edges_out_of = vec![Vec::new(); selectable.executions.len()]; + for (position, edge) in selectable.edges.iter().enumerate() { + selectable.edges_into[edge.consumer][physical.edge(edge.edge).input_index_of_consumer] + .push(position); + selectable.edges_out_of[edge.producer].push(position); + } + selectable + } + + /// The id of `context` with `request_anchor` + /// ([`Self::request_anchor_of_context`]), added to the list when new. + fn context_id( + &mut self, + context: ExecutionContext, + request_anchor: RequestAnchor, + ) -> ExecutionContextId { + let id = ExecutionContextId(self.contexts.len()); + *self + .context_ids + .entry((context, request_anchor.clone())) + .or_insert_with(|| { + self.contexts.push(context); + self.request_anchor_of_context.push(request_anchor); + id + }) + } + + /// The execution of producer `u` a consumer whose inputs run in + /// `context` reads, when `u` delivers on it: its own for an + /// `AcrossEvaluations` producer, the one ingestion context + /// `ExecutionContext::EVERY_ARRIVING_ROW` for another ingestion-time producer, otherwise + /// its run in `context`. + fn execution_reading( + &mut self, + physical: &PhysicalASAPMemoCandidates, + context: ExecutionContextId, + u: PhysicalCandidateId, + ) -> Option { + let producer = &physical.physical_candidates[u.0]; + if !physical.delivers_on(u, &self.contexts[context.0]) { + return None; + } + let properties = producer.physical_properties; + let context = if properties.is_across_evaluations() { + self.context_id(ExecutionContext::Materialized(u), None) + } else if properties.timing() == ExecutionTiming::IngestionTime { + self.context_id(ExecutionContext::EVERY_ARRIVING_ROW, None) + } else { + context + }; + Some(Execution { + context, + candidate: u, + }) + } + + /// The context the inputs of `consumer` run in: an `AcrossEvaluations` + /// candidate's at ingestion time, the ingestion pipeline of its refresh + /// window, the one the installed plan refreshes it on + /// (`RefreshWindow::computed_from` over the candidates: `sum(TimeRange(5m))` + /// every 5 min, a 60 s chain over `TimeShift(30s)` at phase 30 s); a `TimeShift` anchored at + /// `@ end()` or `@ start()` in an evaluation context, a context of its + /// own per anchor, on the grid of its queries' request end (start) + /// times: a request's start and end, and an evaluation, are different + /// times (unless the anchor is the queries' evaluation time, + /// `anchor_is_evaluation_time`); else the consumer's own context. + fn input_context( + &mut self, + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + consumer: Execution, + ) -> ExecutionContextId { + let candidate = &physical.physical_candidates[consumer.candidate.0]; + let group = candidate.logical_candidate_id.group; + let properties = candidate.physical_properties; + if properties.timing() == ExecutionTiming::IngestionTime + && properties.is_across_evaluations() + { + return match RefreshWindow::computed_from(physical, consumer.candidate) { + Some(window) => self.context_id(ExecutionContext::IngestionPipeline(window), None), + None => consumer.context, + }; + } + let (Some(at), ExecutionContext::Evaluation { grid }) = ( + physical.operator(consumer.candidate).request_anchor(), + self.contexts[consumer.context.0], + ) else { + return consumer.context; + }; + let queries: Vec = + queries_of_context(grid.period_ms, self.readers_of(group, |_| true), entries) + .into_iter() + .map(QueryId) + .collect(); + if anchor_is_evaluation_time(at, queries.iter().map(|query| &entries[query.0])) { + return consumer.context; + } + let period_ms = queries.iter().fold(0, |period_ms, query| { + greatest_common_divisor(period_ms, entries[query.0].request_time_grid_ms(at)) + }); + match period_ms { + 0 => consumer.context, + period_ms => self.context_id( + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms: 0, + }, + }, + Some((at, queries)), + ), + } + } + + /// The queries whose walk reaches `group` ([`Self::reads_of_group`]), + /// each once, reading it with an anchor `anchored` keeps. + pub(crate) fn readers_of( + &self, + group: MemoGroupId, + anchored: impl Fn(Option) -> bool, + ) -> BTreeSet { + self.reads_of_group + .get(&group) + .into_iter() + .flatten() + .filter(|(_, anchor)| anchored(*anchor)) + .map(|&(query, _)| query) + .collect() + } + + /// Keeps, of the executions answering each query output, those `keep` + /// keeps (given the query, the execution's position and the + /// execution). + pub(crate) fn retain_answering( + &mut self, + mut keep: impl FnMut(QueryId, usize, Execution) -> bool, + ) { + let executions = &self.executions; + for (query, answering) in &mut self.answering { + answering.retain(|&index| keep(*query, index, executions[index])); + } + } + + /// The evaluation context query `query` is evaluated in + /// ([`Self::evaluation_context_of_query`]); `None` for a query with no + /// output reference. + pub(crate) fn evaluation_context(&self, query: QueryId) -> Option { + self.evaluation_context_of_query + .get(query.0) + .copied() + .flatten() + } + + /// Whether the context with id `id` is a root evaluation context: a + /// query's own (`Evaluation`, no request anchor), not a request + /// context, `Materialized(v)` or an ingestion pipeline. + pub(crate) fn is_root_evaluation_context(&self, id: ExecutionContextId) -> bool { + matches!(self.contexts[id.0], ExecutionContext::Evaluation { .. }) + && self.request_anchor_of_context[id.0].is_none() + } + + /// The context with id `id`. + pub(crate) fn context(&self, id: ExecutionContextId) -> ExecutionContext { + self.contexts[id.0] + } + + /// For the context with id `id` that runs the inputs of a `TimeShift` + /// anchored with `@ end()` or `@ start()`: the anchor, and the queries + /// whose requests it runs for. + pub(crate) fn request_anchor( + &self, + id: ExecutionContextId, + ) -> Option<(AtModifier, &[QueryId])> { + self.request_anchor_of_context[id.0] + .as_ref() + .map(|(at, queries)| (*at, queries.as_slice())) + } + + /// Every execution reached from a query output, ascending. + pub(crate) fn executions(&self) -> &[Execution] { + &self.executions + } + + /// Every $y_{\kappa,u,c,i}$, ascending. + pub(crate) fn edges(&self) -> &[SelectableEdge] { + &self.edges + } + + /// The positions in [`Self::edges`] of the edges out of the execution + /// at position `producer`. + pub(crate) fn edges_out_of(&self, producer: usize) -> &[usize] { + &self.edges_out_of[producer] + } + + /// Each query output reference with the executions, by position, that + /// may answer it, in `Memo::output_groups` order. + pub(crate) fn answering(&self) -> &[(QueryId, Vec)] { + &self.answering + } + + /// Every read of an execution's rows, as (the execution read, by + /// position in [`Self::executions`]; its reader; how far back it reads, + /// in ms, `None` when unbounded): each selectable edge into its + /// producer, `need_ms_per_edge` by edge position + /// (`crate::retention::edge_retention_need_ms`), and each query output + /// into each execution that may answer it, the query's range-evaluation + /// span. Costing ($h_v$ at its largest), (R), the least $h_v$ of a + /// selection and the greedy bound each fold it. + pub(crate) fn reads_back<'a>( + &'a self, + need_ms_per_edge: &'a [Option], + entries: &'a [QueryWorkloadEntry], + ) -> impl Iterator)> + 'a { + let edges = + self.edges().iter().zip(need_ms_per_edge).enumerate().map( + |(position, (edge, &need_ms))| (edge.producer, Reader::Edge(position), need_ms), + ); + let outputs = self.answering().iter().flat_map(move |(query, answering)| { + let need_ms = entries[query.0].range_span_ms(); + answering + .iter() + .map(move |&index| (index, Reader::Output(*query), Some(need_ms))) + }); + edges.chain(outputs) + } + + /// The positions in [`Self::edges`] of the edges into input `input` of + /// the execution at position `consumer`. + pub(crate) fn edges_into(&self, consumer: usize, input: usize) -> &[usize] { + &self.edges_into[consumer][input] + } + + /// The number of inputs of the execution at position `index`. + pub(crate) fn input_count(&self, index: usize) -> usize { + self.edges_into[index].len() } } -/// Whether `edge`'s producer delivers on every evaluation its query-time -/// consumer serves: on the evaluation context of each query reaching the -/// consumer's group ([`PhysicalCandidate::delivers_on`]). The baseline has -/// no other context: an ingestion-time consumer's (its pipeline) is checked -/// with execution contexts. -fn delivers_to_consumer( +/// Per memo group of `physical`, each (query, anchor) whose walk from the +/// groups its outputs read (`outputs`) reaches it ([`query_reads`]), +/// through the inputs of every candidate of a group. +fn reads_of_groups( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], - edge: &PhysicalEdge, -) -> bool { - let producer: &PhysicalCandidate = &physical.physical_candidates[edge.producer_candidate_id.0]; - let consumer = &physical.physical_candidates[edge.consumer_candidate_id.0]; - if consumer.physical_properties.timing() != ExecutionTiming::QueryTime { - return true; - } - physical.logical_candidates.memo.groups()[consumer.logical_candidate_id.group.0 as usize] - .reached_by_queries() + outputs: &[(QueryId, MemoGroupId)], +) -> HashMap)>> { + let memo = &physical.logical_candidates.memo; + let query_count = outputs .iter() - .filter_map(|&query| entries.get(query)) - .all(|entry| producer.delivers_on(physical, &evaluation_context(entry))) + .map(|(query, _)| query.0 + 1) + .max() + .unwrap_or(0); + let mut starts = vec![Vec::new(); query_count]; + for &(query, group) in outputs { + starts[query.0].push(group); + } + let inputs_of = |group: MemoGroupId| -> Vec { + let inputs: BTreeSet = memo.groups()[group.0 as usize] + .candidates + .iter() + .flat_map(|candidate| candidate.expression.operator.children()) + .copied() + .collect(); + inputs.into_iter().collect() + }; + query_reads( + &starts, + inputs_of, + |group| memo.groups()[group.0 as usize].expression.request_anchor(), + |_| false, + entries, + ) } diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index 58e3bc768..e00962a0a 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -391,7 +391,7 @@ pub struct VectorMatch { /// PromQL `@` modifier — pins a selector's evaluation time to an anchor instead /// of the query evaluation time (issue #40). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub enum AtModifier { /// `@ start()` — the query range's start instant. Start, diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 19910108e..7fa100936 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -10,7 +10,7 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::arguments::Source; +use super::arguments::{AtModifier, Source}; use super::asap::ASAPOp; use super::input::OperatorInput; use super::non_asap::NonASAPOp; @@ -45,6 +45,7 @@ impl Operator { } } + /// `self` with `f` applied to each child, in [`Self::children`] order. /// `f` may change the reference type, e.g. from `Rc` to a /// node id. pub fn map_children(&self, f: impl FnMut(&C) -> D) -> Operator { @@ -60,6 +61,145 @@ impl Operator { Operator::ASAP(op) => op.kind_name(), } } + + /// Its tumbling-window width when it is a `SummaryTumblingWindowChain`, + /// in ms; `None` for every other operator. E.g. a chain of 1-minute + /// windows: `Some(60_000)`; `TimeRange(5m)`: `None`. + pub fn chain_tumbling_window_width_ms(&self) -> Option { + match self { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => None, + } + } + + /// The anchor of a `TimeShift` with `@ end()` or `@ start()`, whose + /// input runs once per request; `None` for any other operator (`@ ` + /// included). + pub fn request_anchor(&self) -> Option { + match self { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { + Some(at @ (AtModifier::End | AtModifier::Start)) => Some(at), + Some(AtModifier::Timestamp(_)) | None => None, + }, + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => None, + } + } + + /// Whether each output row keeps its input row's time window, so the + /// window its input covers is also its own: `Filter` and `Project` + /// keep rows and times; `TimeShift` moves every row by one offset but + /// keeps the window's length. E.g. `Filter <- TimeRange(5m)` covers + /// 5 minutes, and `Project <- chain of 1-minute windows` holds + /// 1-minute windows; `Aggregate <- TimeRange(5m)` combines rows, so + /// `false`. + pub fn keeps_input_window(&self) -> bool { + match self { + Operator::NonASAP( + NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, + ) => true, + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => false, + } + } } /// The derivation rules, one set for every kind of input diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 2318fdd8c..49a361f31 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -251,9 +251,13 @@ impl NonASAPOp { } /// Rebuild this operator with `f` applied to every child, including the - /// operator nodes referenced from scalar expressions. Every other field - /// is cloned. `f` may change the reference type, e.g. from - /// `Rc` to a node id. + /// operator nodes referenced from scalar expressions, in + /// [`Self::children`] order (direct inputs first), so a caller may hand + /// out replacements by position. Every other field is cloned. `f` may + /// change the reference type, e.g. from `Rc` to a node id. + /// + /// Example: `Project { child: q, cols: [q.value + scalar(c)] }` calls `f` + /// on `q`, then on `c`. pub(crate) fn map_children(&self, mut f: impl FnMut(&C) -> D) -> NonASAPOp { use NonASAPOp::*; fn map_pred(p: &Predicate, f: &mut impl FnMut(&C) -> D) -> Predicate { @@ -287,13 +291,14 @@ impl NonASAPOp { }, PromqlVectorFromScalar(e) => PromqlVectorFromScalar(e.map_operator_refs(&mut f)), Filter { pred, child } => { - let pred = map_pred(pred, &mut f); + let child = f(child); Filter { - pred, - child: f(child), + pred: map_pred(pred, &mut f), + child, } } Project { cols, child } => { + let child = f(child); let cols = cols .iter() .map(|c| ProjectItem { @@ -301,10 +306,7 @@ impl NonASAPOp { expr: c.expr.map_operator_refs(&mut f), }) .collect(); - Project { - cols, - child: f(child), - } + Project { cols, child } } Aggregate { reduction, @@ -314,6 +316,7 @@ impl NonASAPOp { having, child, } => { + let child = f(child); let filters = filters .iter() .map(|p| p.as_ref().map(|p| map_pred(p, &mut f))) @@ -325,7 +328,7 @@ impl NonASAPOp { output_names: output_names.clone(), filters, having, - child: f(child), + child, } } Join { @@ -334,12 +337,12 @@ impl NonASAPOp { left, right, } => { - let pred = map_pred(pred, &mut f); + let (left, right) = (f(left), f(right)); Join { kind: kind.clone(), - pred, - left: f(left), - right: f(right), + pred: map_pred(pred, &mut f), + left, + right, } } SetOp { @@ -365,11 +368,11 @@ impl NonASAPOp { partition_by, child, } => { - let keys = map_keys(keys, &mut f); + let child = f(child); Sort { - keys, + keys: map_keys(keys, &mut f), partition_by: partition_by.clone(), - child: f(child), + child, } } Limit { @@ -403,6 +406,7 @@ impl NonASAPOp { output_name, child, } => { + let child = f(child); let args = args.iter().map(|e| e.map_operator_refs(&mut f)).collect(); let order_by = map_keys(order_by, &mut f); SQLWindowFunc { @@ -412,7 +416,7 @@ impl NonASAPOp { order_by, frame: frame.clone(), output_name: output_name.clone(), - child: f(child), + child, } } TimeRange { range, kind, child } => TimeRange { @@ -425,11 +429,11 @@ impl NonASAPOp { child: f(child), }, PromqlRelabel { dst, value, child } => { - let value = value.map_operator_refs(&mut f); + let child = f(child); PromqlRelabel { dst: dst.clone(), - value, - child: f(child), + value: value.map_operator_refs(&mut f), + child, } } PromqlInfoEnrich { selector, child } => PromqlInfoEnrich { @@ -1929,4 +1933,111 @@ mod tests { }; assert_eq!(operator.vector_match.as_ref(), Some(&vm)); } + + /// For every operator whose scalar expressions read operator nodes, + /// `map_children` calls `f` in [`NonASAPOp::children`] order (direct + /// inputs, then each node its scalar expressions read, as + /// `ScalarExpr::operator_refs` lists them), so a caller may hand out + /// replacements by position. Example: `Filter` over node 1 with + /// predicate `s(11) IN (SELECT … node 12) AND s(13)` sees 1, 12, 11, + /// 13: an `IN` lists its subquery before the nodes its left side reads. + #[test] + fn map_children_calls_f_in_children_order() { + let read = |id: u32| ScalarExpr::ScalarSubquery(id); + let in_and = |a: u32, b: u32, c: u32| { + ScalarExpr::BoolAnd(vec![ + ScalarExpr::InSubquery { + expr: Box::new(read(a)), + subquery: b, + negated: false, + }, + read(c), + ]) + }; + let key = |id: u32| SortKey { + expr: in_and(id, id + 1, id + 2), + ascending: true, + nulls_first: false, + }; + let empty_schema = Schema { + fields: vec![], + time_index: None, + unique_keys: vec![], + closed: true, + }; + let operators: Vec> = vec![ + NonASAPOp::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![Predicate(in_and(11, 12, 13)), Predicate(read(14))], + schema: empty_schema.clone(), + }, + NonASAPOp::Values { + rows: vec![vec![in_and(11, 12, 13)], vec![read(14)]], + schema: empty_schema, + }, + NonASAPOp::Filter { + pred: Predicate(in_and(11, 12, 13)), + child: 1, + }, + NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: None, + expr: in_and(11, 12, 13), + }, + ProjectItem { + alias: None, + expr: read(14), + }, + ], + child: 1, + }, + NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![], + output_names: vec![], + filters: vec![Some(Predicate(in_and(11, 12, 13))), None], + having: Some(Predicate(read(14))), + child: 1, + }, + NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(in_and(11, 12, 13)), + left: 1, + right: 2, + }, + NonASAPOp::Sort { + keys: vec![key(11), key(14)], + partition_by: GroupKeys::by(vec![]), + child: 1, + }, + NonASAPOp::SQLWindowFunc { + func: WindowFuncKind::RowNumber, + args: vec![in_and(11, 12, 13)], + partition_by: GroupKeys::by(vec![]), + order_by: vec![key(14)], + frame: None, + output_name: "w".into(), + child: 1, + }, + NonASAPOp::PromqlVectorFromScalar(in_and(11, 12, 13)), + NonASAPOp::PromqlRelabel { + dst: "d".into(), + value: in_and(11, 12, 13), + child: 1, + }, + ]; + for operator in &operators { + let mut seen = Vec::new(); + operator.map_children(|&id| { + seen.push(id); + id + }); + let children: Vec = operator.children().into_iter().copied().collect(); + assert_eq!(seen, children, "{operator:?}"); + } + assert_eq!(operators[2].children(), vec![&1, &12, &11, &13]); + } } diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 96debd9e1..02ebeb049 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -3,8 +3,8 @@ //! with a time index. A chain has no range and no window count: a //! `TimeRange` (and `TimeShift`s) over it select the windows a query reads, //! a `SummaryMerge` over that selection is the state of the chain's build -//! over the same range, and how many windows it retains is $h_v$ (not yet -//! specified). +//! over the same range, and how many windows it retains is $h_v$, the +//! longest its selected consumers read back (Stage 3's constraint (R)). use super::asap::ASAPOp; use super::input::OperatorInput; use super::node::Operator; diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 234717316..fe128ffc4 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -263,11 +263,15 @@ impl ScalarExpr { expr, subquery, negated, - } => ScalarExpr::InSubquery { - expr: map_box(expr, column, f), - subquery: f(subquery), - negated: *negated, - }, + } => { + // The subquery first, as `operator_refs` lists it. + let subquery = f(subquery); + ScalarExpr::InSubquery { + expr: map_box(expr, column, f), + subquery, + negated: *negated, + } + } ScalarExpr::Negative { expr, semantics } => ScalarExpr::Negative { expr: map_box(expr, column, f), semantics: *semantics, diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index a00db1e0a..e3a8182ae 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -75,8 +75,9 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use super::physical_asap_dag::{PhysicalASAPDAGValidationError, PhysicalASAPNodeId, ViewCoverage}; -use super::time_window::TimeWindow; +use super::time_window::{self, TimeIndexedPlan, TimeWindow}; use crate::ir::operator::arguments::Source; +use crate::stages::EvaluationGrid; /// One materialized view of a DAG: what its stored state is, what it is /// refreshed from, when, and for how long it is retained. @@ -172,6 +173,40 @@ impl RefreshWindow { phase_ms, }) } + + /// The refresh window of `node`, stored at ingestion time in `plan`: + /// that of the smallest window holding every window its result is + /// computed from ([`time_window::windows_computed_from`]). A + /// `SummaryTumblingWindowChain` of width `w` over shifts summing to `s` + /// is `(w, s mod w)`; any other node takes the union of the windows + /// below it (each `TimeRange`'s, and each chain's reached without + /// passing one). `None` when that window is unbounded (none below it, + /// or an `@` anchor below a window) or empty. One rule for the + /// installed plan ([`PhysicalASAPDAG::refresh_window`]), Stage 2's + /// output (`PhysicalASAPMemoCandidates`, which Stage 3 selects from) + /// and the cost model. + /// + /// Example: `sum(TimeRange(5m))` is `(5m, 0)` whatever grid reads it; a + /// chain of 60 s windows over `TimeShift(30s)` is `(60s, 30s)`. + pub fn computed_from(plan: &P, node: P::Node) -> Option { + time_window::windows_computed_from(plan, node)? + .into_iter() + .reduce(TimeWindow::union) + .and_then(Self::of_window) + } +} + +/// The times a refresh window ends at: every `width_ms`, at `phase_ms`. +/// +/// Example: `RefreshWindow { width_ms: 300_000, phase_ms: 0 }`, the +/// refresh of `sum(TimeRange(5m))`, ends every 5 min at phase 0. +impl From for EvaluationGrid { + fn from(window: RefreshWindow) -> Self { + EvaluationGrid { + period_ms: window.width_ms, + phase_ms: window.phase_ms, + } + } } /// The views refreshed at ingestion that share one refresh window diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index 4ac4be474..707899c13 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -96,6 +96,24 @@ impl PhysicalASAPDAGNode { pub fn is_view(&self) -> bool { matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) && !is_raw_data_scan(self) } + + /// Whether it keeps raw rows stored at ingestion time across + /// evaluations: a `Scan`, or a `TimeShift` without `@` whose input + /// (`input`, its first producer's operator) is a `Scan`. Its readers + /// read the rows it retains; an ingestion-time reader reads one batch + /// of them (`asap_cost::reads_one_batch_of`). + pub fn keeps_raw_rows_at_ingestion(&self, input: Option<&Operator>) -> bool { + self.output_state.timing == ExecutionTiming::IngestionTime + && matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) + && match &self.payload { + Operator::NonASAP(NonASAPOp::Scan { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + shift.at.is_none() + && matches!(input, Some(Operator::NonASAP(NonASAPOp::Scan { .. }))) + } + _ => false, + } + } } impl ViewIndex<'_> { diff --git a/crates/types/src/plan/materialized_view/views.rs b/crates/types/src/plan/materialized_view/views.rs index 5c765e11b..835abdc39 100644 --- a/crates/types/src/plan/materialized_view/views.rs +++ b/crates/types/src/plan/materialized_view/views.rs @@ -74,12 +74,11 @@ impl PhysicalASAPDAG { .filter(|view| view.refresh == MaterializedViewRefresh::AtIngestion) .cloned() .collect(); - let refresh_windows = self.refresh_windows(views.iter().map(|view| view.node_id)); let mut groups: Vec = Vec::new(); let mut group_by_window: HashMap = HashMap::new(); let mut group_by_view: HashMap = HashMap::new(); for view in views { - let refresh_window = refresh_windows[&view.node_id]; + let refresh_window = self.refresh_window(view.node_id); let group = match refresh_window.and_then(|window| group_by_window.get(&window)) { Some(&group) => group, None => { @@ -133,34 +132,10 @@ impl PhysicalASAPDAG { .collect() } - /// The refresh window of `node` ([`RefreshWindow::of_window`]): for a - /// `SummaryTumblingWindowChain` of width `w`, `(w, s mod w)` with `s` - /// the sum of the offsets of the `TimeShift`s below it; for any other - /// node, that of the smallest window holding the windows below it (each - /// `TimeRange`'s, and each chain's reached without passing one). `None` - /// when that window is unbounded (none below it, or an `@` anchor below - /// a `TimeRange`) or empty. O(nodes + edges). + /// The refresh window of `node` ([`RefreshWindow::computed_from`] over + /// this DAG). O(nodes + edges) per window walked. pub fn refresh_window(&self, node: PhysicalASAPNodeId) -> Option { - self.refresh_windows([node])[&node] - } - - /// [`Self::refresh_window`] of each of `nodes`, by id, with the DAG - /// indexed once for all of them. - pub fn refresh_windows( - &self, - nodes: impl IntoIterator, - ) -> HashMap> { - let index = ViewIndex::new(self); - nodes - .into_iter() - .map(|node| { - let refresh_window = index - .windows_computed_from(node) - .and_then(|windows| windows.into_iter().reduce(TimeWindow::union)) - .and_then(RefreshWindow::of_window); - (node, refresh_window) - }) - .collect() + RefreshWindow::computed_from(self, node) } } diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 77fe7eb35..7e91ce4b5 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -110,6 +110,20 @@ pub struct PhysicalASAPDAGNode { pub view_coverage: Option, } +impl PhysicalASAPDAGNode { + /// The nodes this node reads, one per edge into it in a validated DAG, + /// in its payload's input order ([`Operator::children`]): its direct + /// inputs, then each node its scalar expressions refer to, once. + /// Example: `q + scalar(count(q))` as a `Project` over `q` reads `q`, + /// then the count. + pub fn producers(&self) -> Vec { + edge_roles(&self.payload) + .into_iter() + .map(|(producer, _)| producer) + .collect() + } +} + /// A view's aggregate coverage /// ([`AggregateCoverage`](crate::stages::AggregateCoverage), `docs/design_docs/proposals/asap-primitive-schema.md` /// §4.2): what it computes and which rows went in. diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index 4100c5463..a9ab9a4cf 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -7,8 +7,8 @@ //! 2. [`InputTimeAccess`]: how one node reads its time-indexed input, //! which rows ([`InputRows`]) and how much earlier its output rows are //! than those ([`InputTimeAccess::shifts_back_ms`]). -//! 3. [`TimeIndexedPlan`]: a plan the walks run over (plan nodes, Stage 3 -//! candidates or executor nodes): each node's readers, inputs, input +//! 3. [`TimeIndexedPlan`]: a plan the walks run over (plan nodes, Stage 2's +//! output `PhysicalASAPMemoCandidates` or executor nodes): each node's readers, inputs, input //! time access and operator, and which nodes are raw data. //! 4. The walks: up from a time-indexed producer (raw data, a chain, //! results stored per evaluation time) through its readers, the window @@ -158,8 +158,8 @@ impl InputTimeAccess { } } -/// A plan the time walks run over: plan nodes, Stage 3 candidates or -/// executor nodes. +/// A plan the time walks run over: plan nodes, Stage 2's output +/// (`PhysicalASAPMemoCandidates`) or executor nodes. pub trait TimeIndexedPlan { type Node: Copy + Ord; /// The child type of the operators it holds. @@ -285,30 +285,47 @@ pub fn windows_readers_need( Some(windows) } -/// The sum of the offsets of the `TimeShift`s directly below `node` (each -/// the only input of the node above it). `None` when one has an `@` -/// anchor, or the sum is negative or overflows. +/// The sum of the offsets of the `TimeShift`s directly below `node`, each +/// the input of the node above it. Over a plan each of these nodes has one +/// input; over Stage 2's output (`PhysicalASAPMemoCandidates`) its inputs +/// are the alternative producers of its one input, and every path down +/// must give the same sum. `None` when one has an `@` anchor, the paths disagree, or the sum +/// is negative or overflows. /// /// Example: `TimeRange(5m)` over `TimeShift(1m)` over `TimeShift(1h)`: /// 3,660,000. pub fn shift_back_ms_below(plan: &P, node: P::Node) -> Option { - let only_input = |node| match plan.inputs_of(node).as_slice() { - [input] => Some(*input), - _ => None, - }; - let mut shift_back_ms = 0i64; - let mut current = node; - while let Some(input) = only_input(current) { - let Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) = plan.operator_of(input) else { - break; - }; - if shift.at.is_some() { - return None; + let mut shift_back_ms: Option = None; + let mut seen = std::collections::BTreeSet::new(); + // (a node, the offsets of the shifts walked through to it) + let mut pending = vec![(node, 0i64)]; + while let Some((current, walked_ms)) = pending.pop() { + if !seen.insert((current, walked_ms)) { + continue; + } + let inputs = plan.inputs_of(current); + // A path ends at a node with no input, or at an input that is not a + // shift. + let mut path_ends = inputs.is_empty(); + for input in inputs { + match plan.operator_of(input) { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + if shift.at.is_some() { + return None; + } + pending.push((input, walked_ms.checked_add(shift.offset_ms)?)); + } + _ => path_ends = true, + } + } + if path_ends { + if shift_back_ms.is_some_and(|sum| sum != walked_ms) { + return None; + } + shift_back_ms = Some(walked_ms); } - shift_back_ms = shift_back_ms.checked_add(shift.offset_ms)?; - current = input; } - u64::try_from(shift_back_ms).ok() + u64::try_from(shift_back_ms?).ok() } /// Down: the windows of input rows that `node`'s result is computed from: diff --git a/crates/types/src/stages/logical/group/mod.rs b/crates/types/src/stages/logical/group/mod.rs index c460544b8..cf087d8ea 100644 --- a/crates/types/src/stages/logical/group/mod.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -90,7 +90,7 @@ pub struct GroupProperties { /// says nothing about whether rows keep arriving: a result that does /// not depend on the evaluation time is final, the same at every /// evaluation, only once its rows no longer change - /// ([`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on)). + /// ([`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on)). pub depends_on_evaluation_time: bool, /// What an aggregate group covers: a summary state, a chain, a /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact @@ -189,7 +189,7 @@ pub(super) fn derive_group_properties( /// ([`reads_time_not_pinned_by_at`]) still depends on it. Any other /// operator depends on it when an input group does. A `Scan` reads no time: /// whether its rows keep arriving is -/// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on)'s +/// [`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on)'s /// check. fn depends_on_evaluation_time(operator: &Operator>) -> bool { match operator { diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 7c492061e..7d52645b2 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -177,6 +177,25 @@ impl Memo { &self.queries } + /// Each query output reference (the query's index, and the group its + /// operator root, or one operator of its scalar root, reads), + /// deduplicated, ascending. + pub fn output_groups(&self) -> Vec<(usize, MemoGroupId)> { + self.queries + .iter() + .enumerate() + .flat_map(|(index, query)| { + query + .root + .operator_refs() + .into_iter() + .map(move |&group| (index, group)) + }) + .collect::>() + .into_iter() + .collect() + } + /// Adds `query` and returns its index: its root's groups and every /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 /// adds each query once. diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 16125313f..bdb3b9090 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -4,7 +4,22 @@ use std::collections::BTreeSet; -use crate::stages::{EvaluationGrid, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate}; +use crate::stages::{ + CandidateForMemoGroupId, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate, +}; + +/// The times `phase_ms + k · period_ms`: the evaluation times of a +/// context, or the times a result materialized across evaluations is +/// stored at (`PhysicalASAPMemoCandidates::stored_times`). +/// +/// Example: a query every 60 s is evaluated on `{ period_ms: 60_000, +/// phase_ms: 0 }`; a chain of 60 s windows over `TimeShift(30s)` is stored +/// on `{ period_ms: 60_000, phase_ms: 30_000 }`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EvaluationGrid { + pub period_ms: u64, + pub phase_ms: u64, +} /// Stage 1, Pass 2's output. #[derive(Debug, Clone)] @@ -30,14 +45,41 @@ pub struct LogicalASAPMemoCandidates { /// Q0 = `quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, /// m[5m])` and Q1 = the same ratio `> 1`, both relative 0.05: Q0 may /// read the DDSketch readouts, while Q1's comparison has no contract, - /// so its set holds the ratio group and every group below it. (Both + /// so its set holds the ratio group and every group below it, except + /// below an input whose error does not reach its consumer's output (a + /// `count`'s input: an exact `count(q)` lists the count, not `q`). (Both /// list the rows below a sketch build, which has no contract.) A query /// whose whole-result failure probability has no per-value one ($n_q$ /// unknown) also lists each readout group another query gave a level, /// with the groups below it. A group is listed for a query, not /// pruned: other queries may still read its approximate candidates, so - /// Stage 3 must enforce it per query, in each query's selection - /// context; Pass 2 only lists it. Empty for a query without an accuracy + /// Stage 3 enforces it per query, in each query's selection context + /// (its MILP's constraint (X)); Pass 2 only lists it. Empty for a query without an accuracy /// target. pub exact_only_groups_of_query: Vec>, } + +impl LogicalASAPMemoCandidates { + /// Whether query `query` must not read `candidate`: it is not exact + /// (it has sketch parameters θ) and its group is one the query may read + /// only exactly ([`Self::exact_only_groups_of_query`]). A readout's own + /// candidate has no θ, but the sketch states it reads are in the same + /// list, as every group below a listed one is, except below an input + /// whose error does not reach its consumer's output (a `count`'s; a + /// readout's state input is not one). + /// + /// Example: `quantile_over_time(0.9, m[5m])` at a relative target lists + /// the quantile and every group below it; its KLL state builds (θ) are + /// candidates it must not read, the exact quantile one it may. + pub fn query_must_not_read(&self, query: usize, candidate: CandidateForMemoGroupId) -> bool { + self.exact_only_groups_of_query + .get(query) + .is_some_and(|groups| groups.contains(&candidate.group)) + && self + .memo + .candidate(candidate) + .logical_properties + .parameters() + .is_some() + } +} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index fe018bb8d..a122a1e55 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -26,5 +26,5 @@ pub use group::{ GroupProperties, MemoGroup, MemoGroupError, MemoGroupId, ValueSign, }; pub use memo::{Memo, MemoQuery}; -pub use memo_candidates::LogicalASAPMemoCandidates; +pub use memo_candidates::{EvaluationGrid, LogicalASAPMemoCandidates}; pub use requirements::{LogicalRequirements, RequiredAccuracy}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 1c70ea118..9b3ba2678 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -16,16 +16,16 @@ mod pruning; pub use logical::{ listed_build_parameters, ranked_column_of_group, AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, - ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, - LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, + ColumnLineage, CoverageError, EvaluationGrid, GroupProperties, LogicalASAPMemoCandidates, + LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, }; pub use physical::{ - output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, - ExecutionContextId, Lifetime, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PhysicalProperties, PhysicalPropertiesError, - StorageMedium, + anchor_is_evaluation_time, producer_satisfies_physical_requirements, queries_of_context, + query_reads, ExecutionContext, ExecutionContextId, Lifetime, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PhysicalProperties, + PhysicalPropertiesError, StorageMedium, }; pub use pruning::{ PruneCause, PruneReason, PrunedCandidate, PrunedLogicalCandidate, RootPruneReason, diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index 8c56560c7..2c345e8e2 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -2,10 +2,8 @@ //! valid $p^{\mathrm{phys}} = (\tau, \ell)$. use crate::ir::properties::ExecutionTiming; -use crate::ir::{ASAPOp, Operator}; -use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; +use crate::stages::CandidateForMemoGroupId; use crate::stages::{Lifetime, PhysicalProperties}; -use crate::workload::DataArrival; /// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -20,96 +18,9 @@ pub struct PhysicalCandidate { pub physical_properties: PhysicalProperties, } -impl PhysicalCandidate { - /// Whether a consumer running in `context` can read this producer (the - /// concepts doc's alignment table, section 2): true for $\ell$ = None, - /// which runs in the consumer's own context; for `OneEvaluation` only in - /// an `Evaluation` context, the one evaluation its result belongs to. - /// With $\ell$ = `AcrossEvaluations`, every execution time of the - /// context must be a time the producer has stored: for a chain - /// (`SummaryTumblingWindowChain`) of width $w$, $w$ divides an - /// evaluation grid's period and phase, or a refresh window's width and - /// phase. Any other result is stored on its group's evaluation grid - /// (`evaluation_grid_of_group`), which must hold every time of the - /// context, unless it is final: the same at every time it is read. - /// - /// Finality assumes no row arrives out of order: at evaluation time t, - /// every row with `ts <= t` has arrived and never changes (#1298 adds - /// a declared lateness). A stored result is final when its group does - /// not depend on the evaluation time - /// ([`GroupProperties::depends_on_evaluation_time`](crate::stages::GroupProperties::depends_on_evaluation_time)) - /// and the rows it read never change after it is computed: - /// - /// | Result | Delivers on | - /// |---|---| - /// | kept up to date at ingestion time (`IngestionTime`) | every context | - /// | over data at rest (`DataArrival::AtRest`) or reading no source | every context | - /// | at query time, reading a time relative to its evaluation (`rate(x[5m])`, `@ end()`, `now()`) | its own grid | - /// | at query time over arriving rows, no time bound (SQL `count(*)`) | its own grid: never final | - /// | at query time over arriving rows, a window ending at E (`sum_over_time(x[5m] @ T)`, `ts BETWEEN a AND b`) | its own grid: final only from E on, and a context carries no absolute time to compare with E | - /// - /// `DataArrival::Unknown` counts as arriving. - pub fn delivers_on( - &self, - candidates: &PhysicalASAPMemoCandidates, - context: &ExecutionContext, - ) -> bool { - match self.physical_properties.lifetime() { - Lifetime::None => true, - Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), - Lifetime::AcrossEvaluations(_) => { - let logical = &candidates.logical_candidates; - let group = self.logical_candidate_id.group; - let chain_width_ms = match logical - .memo - .candidate(self.logical_candidate_id) - .expression - .operator - { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(tumbling_window_width_ms), - _ => None, - }; - let group_properties = &logical.memo.groups()[group.0 as usize].group_properties; - let final_at_every_time = !group_properties.depends_on_evaluation_time - && (self.physical_properties.timing() == ExecutionTiming::IngestionTime - || logical.memo.data_arrival() == DataArrival::AtRest - || group_properties.scanned_sources.is_empty()); - if chain_width_ms.is_none() && final_at_every_time { - return true; - } - let (period_ms, phase_ms) = match context { - ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), - ExecutionContext::IngestionPipeline(window) => { - (window.width_ms, window.phase_ms) - } - ExecutionContext::Materialized(_) => { - todo!( - "#1207 PR 7: the times of a Materialized(v) context, v's own schedule" - ) - } - }; - match chain_width_ms { - Some(width_ms) => { - period_ms.is_multiple_of(width_ms) && phase_ms.is_multiple_of(width_ms) - } - None => { - logical.evaluation_grid_of_group[group.0 as usize].is_some_and(|grid| { - period_ms.is_multiple_of(grid.period_ms) - && phase_ms % grid.period_ms == grid.phase_ms % grid.period_ms - }) - } - } - } - } - } -} - /// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts /// doc's table, section 2). Alignment is checked per context in Stage 3 with -/// [`PhysicalCandidate::delivers_on`]. +/// [`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on). pub fn producer_satisfies_physical_requirements( producer: &PhysicalCandidate, consumer_timing: ExecutionTiming, diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs index 31f95c13f..e3b238036 100644 --- a/crates/types/src/stages/physical/execution_context.rs +++ b/crates/types/src/stages/physical/execution_context.rs @@ -1,10 +1,14 @@ //! $\kappa$: a class of executions that happen at the same times (concepts //! doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and $y_{\kappa,u,c,i}$ by -//! it; [`super::PhysicalCandidate::delivers_on`] reads it, so it lives with +//! it; [`super::PhysicalASAPMemoCandidates::delivers_on`] reads it, so it lives with //! the physical candidates. use super::candidate::PhysicalCandidateId; +use crate::ir::operator::AtModifier; use crate::plan::materialized_view::RefreshWindow; +use crate::stages::EvaluationGrid; +use crate::workload::{QueryWorkloadEntry, ONE_SECOND_MS}; +use std::collections::{BTreeSet, HashMap, HashSet}; /// Index of an [`ExecutionContext`] in the workload's list of distinct /// contexts. @@ -26,9 +30,143 @@ pub enum ExecutionContext { Materialized(PhysicalCandidateId), } -/// The evaluation times `phase_ms + k · period_ms`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct EvaluationGrid { - pub period_ms: u64, - pub phase_ms: u64, +impl ExecutionContext { + /// The one context of every ingestion-time execution that is not + /// materialized: each second of arriving rows, which it computes once + /// for every ingestion-time view of every refresh window that reads it. + pub const EVERY_ARRIVING_ROW: Self = Self::IngestionPipeline(RefreshWindow { + width_ms: ONE_SECOND_MS, + phase_ms: 0, + }); + + /// Whether this context runs outside every evaluation context, so + /// every evaluation context reading its executions shares them: the + /// ingestion pipeline and a candidate's own materialized computation. + /// + /// Example: a chain materialized at ingestion time, read by queries + /// every 60 s and every 300 s: its `IngestionPipeline` execution is one + /// for both, true; an `Evaluation` context: false. + pub fn is_outside_evaluations(&self) -> bool { + match self { + Self::Evaluation { .. } => false, + Self::IngestionPipeline(_) | Self::Materialized(_) => true, + } + } +} + +/// Whether a `TimeShift` anchored with `at`, read by `queries` (those on +/// one evaluation grid that reach it), is anchored at their evaluation +/// times, so its input runs in their evaluation context, shared with +/// their other reads: `@ end()` with an instant query among them (an +/// instant query's request ends at its evaluation; the input computed at +/// every evaluation of the grid serves the range queries' request ends +/// too). Otherwise it runs once per request, in a context of its own. +/// +/// `@ start()` of an instant query is its evaluation time too, but it +/// keeps a request context of its own: only `@ end()` is taken as the +/// evaluation time. +/// +/// Example: `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, +/// both instant every 60 s: true, so `m[5m]` is computed once per minute +/// for both; the same `@ end()` in a range query alone: false. +pub fn anchor_is_evaluation_time<'e>( + at: AtModifier, + queries: impl IntoIterator, +) -> bool { + at == AtModifier::End + && queries.into_iter().any(|entry| { + entry + .range_evaluation + .is_none_or(|range| range.evaluations_per_request() <= 1) + }) +} + +/// The queries of the execution context on evaluation grid `grid_ms` +/// among `reaching` (the queries reaching a node, by index in `entries`): +/// those evaluated on that grid +/// ([`QueryWorkloadEntry::evaluation_grid_ms`]; 1 ms for a query without +/// an entry). Stage 3 (over memo groups) and `asap_cost`'s `computations` +/// (over plan nodes) pick a context's queries with it. +pub fn queries_of_context( + grid_ms: u64, + reaching: impl IntoIterator, + entries: &[QueryWorkloadEntry], +) -> Vec { + reaching + .into_iter() + .filter(|&query| { + entries + .get(query) + .map_or(1, QueryWorkloadEntry::evaluation_grid_ms) + == grid_ms + }) + .collect() +} + +/// How each query reads the nodes below it, in a plan or a memo: per node, +/// each (query, anchor) whose walk from the query's `starts` down +/// `inputs_of` reaches it, once, skipping (and stopping at) a node +/// `stops_at` keeps. Below a `TimeShift` anchored with `@ end()` or +/// `@ start()` (`anchor_of`, `Operator::request_anchor`) a query reads per request, with that anchor, +/// unless the anchor is its evaluation time ([`anchor_is_evaluation_time`] +/// over the queries on its grid that reach the shift through any node); +/// elsewhere with none. Stage 3 walks memo groups with it and +/// `asap_cost`'s `consumers` plan nodes, so both rate a context by the queries whose +/// walk reaches it. Stage 3 walks every candidate of a group, so a query +/// reading a group through one candidate counts for the group's other +/// candidates too. +/// +/// Example: an instant `sum_over_time(m[5m] @ end())` and a range +/// `sum_over_time(m[5m] @ end() offset 10m)`, both every 60 s: `m[5m]` is +/// read by the instant query with no anchor and by the range query with +/// `@ end()` (its requests' ends are not its evaluations). +pub fn query_reads( + starts: &[Vec], + inputs_of: impl Fn(N) -> Vec, + anchor_of: impl Fn(N) -> Option, + stops_at: impl Fn(N) -> bool, + entries: &[QueryWorkloadEntry], +) -> HashMap)>> { + // The queries reaching each anchored shift, through any node. + let mut reaching_shift: HashMap> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { + let mut seen = HashSet::new(); + let mut stack = start.clone(); + while let Some(id) = stack.pop() { + if seen.insert(id) { + if anchor_of(id).is_some() { + reaching_shift.entry(id).or_default().insert(query); + } + stack.extend(inputs_of(id)); + } + } + } + let in_evaluation_context = |shift: N, query: usize, at: AtModifier| { + let Some(entry) = entries.get(query) else { + return false; + }; + let on_grid = queries_of_context( + entry.evaluation_grid_ms(), + reaching_shift.get(&shift).into_iter().flatten().copied(), + entries, + ); + anchor_is_evaluation_time(at, on_grid.iter().map(|&q| &entries[q])) + }; + let mut reads: HashMap)>> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { + let mut seen = HashSet::new(); + let mut stack: Vec<(N, Option)> = start.iter().map(|&id| (id, None)).collect(); + while let Some((id, anchor)) = stack.pop() { + if stops_at(id) || !seen.insert((id, anchor)) { + continue; + } + reads.entry(id).or_default().push((query, anchor)); + let below = match (anchor, anchor_of(id)) { + (None, Some(at)) if !in_evaluation_context(id, query, at) => Some(at), + (anchor, _) => anchor, + }; + stack.extend(inputs_of(id).into_iter().map(|input| (input, below))); + } + } + reads } diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs index e799cc98f..c52541a0f 100644 --- a/crates/types/src/stages/physical/memo_candidates/mod.rs +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -6,11 +6,15 @@ use std::collections::HashMap; -use crate::ir::QueryRoot; +use crate::ir::properties::ExecutionTiming; +use crate::ir::{NonASAPOp, Operator}; +use crate::plan::materialized_view::RefreshWindow; +use crate::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use crate::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, PrunedCandidate, + EvaluationGrid, ExecutionContext, Lifetime, LogicalASAPMemoCandidates, MemoGroupId, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PrunedCandidate, }; +use crate::workload::DataArrival; /// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] @@ -78,16 +82,20 @@ impl PhysicalASAPMemoCandidates { &self.edges[id.0] } - /// The number of inputs of candidate `v`: its logical candidate's - /// operator's children (e.g. 2 for a binary operator). - pub fn input_count(&self, v: PhysicalCandidateId) -> usize { - self.logical_candidates + /// The operator of candidate `v`'s logical candidate. + pub fn operator(&self, v: PhysicalCandidateId) -> &Operator { + &self + .logical_candidates .memo .candidate(self.physical_candidates[v.0].logical_candidate_id) .expression .operator - .children() - .len() + } + + /// The number of inputs of candidate `v`: its logical candidate's + /// operator's children (e.g. 2 for a binary operator). + pub fn input_count(&self, v: PhysicalCandidateId) -> usize { + self.operator(v).children().len() } /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input @@ -107,24 +115,148 @@ impl PhysicalASAPMemoCandidates { pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { &self.admissible_edges_out_of_candidate[producer.0] } -} -/// The output references of `query_roots`: each operator reference, in -/// order (a scalar root's in `map_operator_refs` order), with the index of -/// the query whose root it is. -pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, C)> { - let mut references = Vec::new(); - for (query, root) in query_roots.iter().enumerate() { - match root { - QueryRoot::Operator(reference) => references.push((query, reference.clone())), - QueryRoot::Scalar(expression) => { - expression.map_operator_refs(&mut |reference| { - references.push((query, reference.clone())) - }); + /// Whether candidate `v` is a `Scan` run at query time, which reads raw + /// data in each execution (a current-time read of it needs every row, + /// [`TimeIndexedPlan::current_time_read_needs_every_row`]). + /// + /// Example: a query-time `Scan` of `latency`: true; the same scan kept + /// at ingestion time, or a `TimeRange(5m)` over it: false. + pub fn is_query_time_scan(&self, v: PhysicalCandidateId) -> bool { + self.physical_candidates[v.0].physical_properties.timing() == ExecutionTiming::QueryTime + && self.current_time_read_needs_every_row(v) + } + + /// The times candidate `v`, materialized across evaluations, has a + /// stored result for, as a grid. A chain, at either timing, and any + /// result kept at ingestion time are stored at the end of every + /// refresh window, by the rule the installed plan refreshes it on + /// ([`RefreshWindow::computed_from`] over these candidates, width as the + /// period); any other result computed at query time on its group's + /// evaluation grid + /// ([`LogicalASAPMemoCandidates::evaluation_grid_of_group`]). `None` + /// when not known. + /// + /// Example: `sum(TimeRange(5m))` kept at ingestion time is stored every + /// 5 min at phase 0, even when its group is read every 60 s; computed + /// at query time and read every 60 s, every 60 s. A chain of 60 s + /// windows over `TimeShift(30s)` every 60 s at phase 30 s. + pub fn stored_times(&self, v: PhysicalCandidateId) -> Option { + let candidate = &self.physical_candidates[v.0]; + let is_chain = self.operator(v).chain_tumbling_window_width_ms().is_some(); + match candidate.physical_properties.timing() { + ExecutionTiming::QueryTime if !is_chain => { + self.logical_candidates.evaluation_grid_of_group + [candidate.logical_candidate_id.group.0 as usize] + } + ExecutionTiming::QueryTime | ExecutionTiming::IngestionTime => { + RefreshWindow::computed_from(self, v).map(EvaluationGrid::from) } } } - references + + /// Whether a consumer running in `context` can read producer `v` (the + /// concepts doc's alignment table, section 2): true for $\ell$ = None, + /// which runs in the consumer's own context; for `OneEvaluation` only in + /// an `Evaluation` context, the one evaluation its result belongs to. + /// With $\ell$ = `AcrossEvaluations`, every execution time of the + /// context must be a time the producer has a stored result for + /// ([`Self::stored_times`]): its period divides the context's period (or + /// refresh-window width) and the phases agree modulo it, unless the + /// result is final: the same at every time it is read. + /// + /// Finality assumes no row arrives out of order: at evaluation time t, + /// every row with `ts <= t` has arrived and never changes. A stored result that is not a chain is final + /// when its group does not depend on the evaluation time + /// ([`GroupProperties::depends_on_evaluation_time`](crate::stages::GroupProperties::depends_on_evaluation_time)) + /// and the rows it read never change after it is computed: + /// + /// | Result | Delivers on | + /// |---|---| + /// | kept up to date at ingestion time (`IngestionTime`) | every context | + /// | over data at rest (`DataArrival::AtRest`) or reading no source | every context | + /// | at query time, reading a time relative to its evaluation (`rate(x[5m])`, `@ end()`, `now()`) | its own grid | + /// | at query time over arriving rows, no time bound (SQL `count(*)`) | its own grid: never final | + /// | at query time over arriving rows, a window ending at E (`sum_over_time(x[5m] @ T)`, `ts BETWEEN a AND b`) | its own grid: final only from E on, and a context carries no absolute time to compare with E | + /// + /// `DataArrival::Unknown` counts as arriving. The times of a + /// `Materialized(u)` context are $u$'s stored times; when they are not + /// known, no non-final producer delivers on it. + /// + /// Example: `sum(TimeRange(5m))` kept at ingestion time delivers on an + /// evaluation grid every 5 min, not on one every 60 s. + pub fn delivers_on(&self, v: PhysicalCandidateId, context: &ExecutionContext) -> bool { + let producer = &self.physical_candidates[v.0]; + match producer.physical_properties.lifetime() { + Lifetime::None => true, + Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), + Lifetime::AcrossEvaluations(_) => { + let memo = &self.logical_candidates.memo; + let group_properties = + &memo.groups()[producer.logical_candidate_id.group.0 as usize].group_properties; + let final_at_every_time = !group_properties.depends_on_evaluation_time + && (producer.physical_properties.timing() == ExecutionTiming::IngestionTime + || memo.data_arrival() == DataArrival::AtRest + || group_properties.scanned_sources.is_empty()); + if self.operator(v).chain_tumbling_window_width_ms().is_none() + && final_at_every_time + { + return true; + } + let times = match *context { + ExecutionContext::Evaluation { grid } => grid, + ExecutionContext::IngestionPipeline(window) => EvaluationGrid::from(window), + ExecutionContext::Materialized(u) => match self.stored_times(u) { + Some(grid) => grid, + None => return false, + }, + }; + self.stored_times(v).is_some_and(|stored| { + stored.period_ms > 0 + && times.period_ms.is_multiple_of(stored.period_ms) + && times.phase_ms % stored.period_ms == stored.phase_ms % stored.period_ms + }) + } + } + } +} + +/// The candidates as the time walks see them: each candidate's admissible +/// producers as its inputs and admissible consumers as its readers, its +/// operator at its timing and lifetime ([`InputTimeAccess::of`]), and a +/// current-time read of a `Scan` (raw data) needing every row. The +/// producers of one input are alternatives, so a walk down over them +/// covers every plan that may select them. +impl TimeIndexedPlan for PhysicalASAPMemoCandidates { + type Node = PhysicalCandidateId; + type Child = MemoGroupId; + + fn readers_of(&self, v: PhysicalCandidateId) -> Vec { + self.admissible_edges_out_of(v) + .iter() + .map(|&out| self.edge(out).consumer_candidate_id) + .collect() + } + + fn inputs_of(&self, v: PhysicalCandidateId) -> Vec { + (0..self.input_count(v)) + .flat_map(|input| self.admissible_edges_into(v, input)) + .map(|&edge| self.edge(edge).producer_candidate_id) + .collect() + } + + fn input_time_access(&self, v: PhysicalCandidateId) -> InputTimeAccess { + let properties = self.physical_candidates[v.0].physical_properties; + InputTimeAccess::of(self.operator(v), properties.timing(), properties.lifetime()) + } + + fn current_time_read_needs_every_row(&self, v: PhysicalCandidateId) -> bool { + matches!(self.operator(v), Operator::NonASAP(NonASAPOp::Scan { .. })) + } + + fn operator_of(&self, v: PhysicalCandidateId) -> &Operator { + self.operator(v) + } } #[cfg(test)] diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 2685a3d28..14a3ee2fa 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -23,7 +23,7 @@ use crate::ir::{NonASAPOp, TimeRangeKind}; use crate::stages::StorageMedium; -use crate::stages::physical::execution_context::EvaluationGrid; +use crate::stages::EvaluationGrid; use crate::stages::{ CandidateForMemoGroup, LogicalASAPMemoCandidates, Memo, MemoGroupId, @@ -150,6 +150,19 @@ fn candidate( } } +/// [`PhysicalASAPMemoCandidates::delivers_on`] of `producer`, added as the +/// last candidate of `candidates`. +fn delivers_on( + candidates: &PhysicalASAPMemoCandidates, + producer: &PhysicalCandidate, + context: &ExecutionContext, +) -> bool { + let mut with_producer = candidates.clone(); + with_producer.physical_candidates.push(*producer); + let v = PhysicalCandidateId(with_producer.physical_candidates.len() - 1); + with_producer.delivers_on(v, context) +} + fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { ExecutionContext::Evaluation { grid: EvaluationGrid { @@ -183,7 +196,7 @@ fn delivers_on_by_lifetime_without_windows() { (&one_evaluation, materialized, false), ] { assert_eq!( - producer.delivers_on(&candidates, &context), + delivers_on(&candidates, producer, &context), delivers, "{:?} in {context:?}", producer.physical_properties @@ -219,7 +232,7 @@ fn delivers_on_a_chain_when_its_width_divides_the_context_times() { (&ingestion_10s, ingestion(5_000, 0), false), ] { assert_eq!( - producer.delivers_on(&candidates, &context), + delivers_on(&candidates, producer, &context), delivers, "{:?} {:?} in {context:?}", producer.logical_candidate_id, @@ -251,7 +264,11 @@ fn delivers_on_a_stored_result_on_its_group_grid() { (&on_grid, ingestion(60_000, 0), true), (&without_grid, evaluation(60_000, 0), false), ] { - assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + assert_eq!( + delivers_on(candidates, &x, &context), + delivers, + "{context:?}" + ); } } @@ -377,8 +394,8 @@ fn delivers_on_grid_and_off_it( ) -> (bool, bool) { let stored = candidate(producer, timing, Lifetime::AcrossEvaluations(MEMORY)); ( - stored.delivers_on(&examples.candidates, &evaluation(60_000, 0)), - stored.delivers_on(&examples.candidates, &evaluation(30_000, 0)), + delivers_on(&examples.candidates, &stored, &evaluation(60_000, 0)), + delivers_on(&examples.candidates, &stored, &evaluation(30_000, 0)), ) } @@ -410,6 +427,36 @@ fn a_count_over_arriving_rows_computed_at_query_time_delivers_on_its_own_grid_on ); } +/// A view kept up to date at ingestion time with no window (SQL +/// `count(*)` over the arriving rows, stored across evaluations) folds in +/// every arriving batch, so it is current at every time: it delivers on +/// every evaluation grid and every ingestion pipeline, whatever its period +/// and phase, not only on its group's 60 s grid. +#[test] +fn an_ingestion_time_view_without_a_window_delivers_on_every_context() { + for arrival in ARRIVING { + let examples = finality_examples(arrival); + let stored = candidate( + examples.count_of_table, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for context in [ + evaluation(60_000, 0), + evaluation(30_000, 0), + evaluation(45_000, 15_000), + evaluation(7_000, 3_000), + ingestion(1_000, 0), + ingestion(60_000, 30_000), + ] { + assert!( + delivers_on(&examples.candidates, &stored, &context), + "{arrival:?} {context:?}" + ); + } + } +} + /// Case (b) of #1264: in `max_over_time((rate(latency[5m] @ end()))[10m:1m] /// @ T)`, `@ T` pins only the subquery's own time; the inner `@ end()` /// still moves with the request. Even over data at rest, a stored result @@ -472,6 +519,70 @@ fn a_window_pinned_by_at_t_delivers_on_every_context_once_nothing_changes_it() { } } +/// In `Materialized(v)` a producer delivers on $v$'s own times: a +/// 5 s chain inside the computation of a 10 s chain (every 10 s) and of +/// X = the scan's last 5 minutes stored on its 60 s grid, but not a +/// 10 s chain inside X's computation every 25 s, nor anything +/// time-dependent inside a build with neither a width nor a grid. +#[test] +fn delivers_on_the_own_times_of_a_materialized_computation() { + let grid = |period_ms| { + Some(EvaluationGrid { + period_ms, + phase_ms: 0, + }) + }; + let materialized = |grid_of_x| { + let mut candidates = candidates(memo(), vec![None, None, None, None, grid_of_x]); + candidates.physical_candidates = vec![ + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + candidate(BUILD, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + ]; + candidates + }; + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let across_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + let in_computation_of = |v| ExecutionContext::Materialized(PhysicalCandidateId(v)); + for (candidates, producer, context, delivers) in [ + ( + materialized(grid(60_000)), + &across_5s, + in_computation_of(0), + true, + ), + ( + materialized(grid(60_000)), + &across_5s, + in_computation_of(1), + true, + ), + ( + materialized(grid(25_000)), + &across_10s, + in_computation_of(1), + false, + ), + ( + materialized(grid(60_000)), + &across_5s, + in_computation_of(2), + false, + ), + ] { + assert_eq!( + delivers_on(&candidates, producer, &context), + delivers, + "{:?} in {context:?}", + producer.logical_candidate_id + ); + } +} + /// `producer_satisfies_physical_requirements` over every valid producer state and every valid /// consumer state: a query-time producer serves only query-time /// consumers, an ingestion-time producer that is not materialized only @@ -607,3 +718,124 @@ fn new_indexes_only_admissible_edges() { .is_empty()); } } + +/// Stored at ingestion time, a result is stored at the ends of the refresh +/// windows the installed plan refreshes it on +/// ([`RefreshWindow::computed_from`]), not on its group's evaluation grid: +/// `sum(TimeRange(5m))` read every 60 s is refreshed every 5 min, so it +/// delivers on a 5 min grid but not on the 60 s one. A chain of 60 s +/// windows over `TimeShift(30s)` is stored at phase 30 s at either timing +/// (its windows end 30 s before the evaluation time); computed at query +/// time, `sum(TimeRange(5m))` is stored on its group's 60 s grid. +#[test] +fn a_stored_chain_and_an_ingestion_time_result_are_stored_on_their_refresh_window() { + let mut memo = Memo::new(&DataWorkload::default()); + let mut add = |operator: Operator, parameters| { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let candidate = CandidateForMemoGroup::new( + operator, + parameters, + OriginOfCandidateForMemoGroup::Pass1, + vec![], + ); + memo.add_candidate(group, candidate).unwrap(); + CandidateForMemoGroupId { + group, + index_in_group: 0, + } + }; + let series = add(scan("latency", &[], vec![]), None); + let last_5m = add(range(300_000, series.group), None); + let sum = add( + aggregate( + Reduction::PerEntity, + AggIntent::Sum { col: Some(1) }, + last_5m.group, + ), + None, + ); + let shifted = add(shift(30_000, None, series.group), None); + let chain = add( + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: shifted.group, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }), + Some(SketchParams::Kll { k: 269 }), + ); + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let group_count = memo.groups().len(); + let stored = Lifetime::AcrossEvaluations(MEMORY); + let candidates_at = |timing: ExecutionTiming| { + PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo: memo.clone(), + edges: vec![], + pruned: vec![], + evaluation_grid_of_group: vec![grid_60s; group_count], + exact_only_groups_of_query: vec![], + }, + vec![ + candidate(series, timing, Lifetime::None), + candidate(last_5m, timing, Lifetime::None), + candidate(sum, timing, stored), + candidate(shifted, timing, Lifetime::None), + candidate(chain, timing, stored), + ], + vec![ + physical_edge(0, 1, true, true), + physical_edge(1, 2, true, true), + physical_edge(0, 3, true, true), + physical_edge(3, 4, true, true), + ], + vec![], + ) + }; + let (sum, chain) = (PhysicalCandidateId(2), PhysicalCandidateId(4)); + let ingestion_time = candidates_at(IngestionTime); + assert_eq!( + RefreshWindow::computed_from(&ingestion_time, sum), + Some(RefreshWindow { + width_ms: 300_000, + phase_ms: 0 + }) + ); + let chain_stored = EvaluationGrid { + period_ms: 60_000, + phase_ms: 30_000, + }; + let query_time = candidates_at(QueryTime); + assert_eq!(ingestion_time.stored_times(chain), Some(chain_stored)); + assert_eq!(query_time.stored_times(chain), Some(chain_stored)); + assert_eq!(query_time.stored_times(sum), grid_60s); + for (candidates, producer, context, delivers) in [ + (&ingestion_time, sum, evaluation(60_000, 0), false), + (&ingestion_time, sum, evaluation(300_000, 0), true), + (&ingestion_time, sum, evaluation(600_000, 300_000), true), + (&ingestion_time, sum, ingestion(300_000, 0), true), + (&ingestion_time, chain, evaluation(60_000, 0), false), + (&ingestion_time, chain, evaluation(60_000, 30_000), true), + (&ingestion_time, chain, evaluation(120_000, 90_000), true), + (&ingestion_time, chain, ingestion(60_000, 0), false), + (&query_time, sum, evaluation(60_000, 0), true), + (&query_time, sum, evaluation(60_000, 30_000), false), + (&query_time, chain, evaluation(60_000, 0), false), + (&query_time, chain, evaluation(60_000, 30_000), true), + (&query_time, chain, evaluation(120_000, 90_000), true), + ] { + assert_eq!( + candidates.delivers_on(producer, &context), + delivers, + "{producer:?} in {context:?} at {:?}", + candidates.physical_candidates[producer.0] + .physical_properties + .timing() + ); + } +} diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 2b3728d4d..c07a8627a 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -18,6 +18,9 @@ pub use candidate::{ producer_satisfies_physical_requirements, PhysicalCandidate, PhysicalCandidateId, }; pub use edge::{PhysicalEdge, PhysicalEdgeId}; -pub use execution_context::{EvaluationGrid, ExecutionContext, ExecutionContextId}; -pub use memo_candidates::{output_references, PhysicalASAPMemoCandidates}; +pub use execution_context::{ + anchor_is_evaluation_time, queries_of_context, query_reads, ExecutionContext, + ExecutionContextId, +}; +pub use memo_candidates::PhysicalASAPMemoCandidates; pub use properties::{Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium}; diff --git a/crates/types/src/stages/physical/properties/mod.rs b/crates/types/src/stages/physical/properties/mod.rs index e37dad27d..530df3ace 100644 --- a/crates/types/src/stages/physical/properties/mod.rs +++ b/crates/types/src/stages/physical/properties/mod.rs @@ -86,6 +86,21 @@ impl PhysicalProperties { pub fn is_materialized(&self) -> bool { self.lifetime != Lifetime::None } + + /// Whether $\ell_v$ = `AcrossEvaluations`: its result is kept from one + /// evaluation to the next. Derived, not stored. + pub fn is_across_evaluations(&self) -> bool { + matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) + } + + /// Whether it has at most one consumer per execution context: a + /// query-time result that is not materialized ($\ell$ = None). + /// Consumers in one context that share a result read a `OneEvaluation` + /// or a materialized producer; the consumers of one ingestion run share + /// its rows. + pub fn has_one_reader_per_context(&self) -> bool { + self.lifetime == Lifetime::None && self.timing == ExecutionTiming::QueryTime + } } #[cfg(test)] diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 42d16a762..93385c36e 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -7,6 +7,12 @@ use serde::{Deserialize, Serialize}; use super::{DurationMs, WorkloadError}; +/// One second, in ms: the unit of every per-second rate +/// ([`DataArrival::rows_per_second`]). An ingestion-time node runs over +/// the rows arriving in one second, and an ingestion-time view that +/// maintains no window is refreshed once per second. +pub const ONE_SECOND_MS: u64 = 1_000; + /// How the queried data arrives: already stored, still arriving, or both. /// This is independent of whether queries repeat. /// diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index 2af8e5966..cff20a85c 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -11,7 +11,7 @@ mod query; pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, - StreamNorms, + StreamNorms, ONE_SECOND_MS, }; pub use planning::{PlanningWorkload, WorkloadError}; pub use query::{ diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index d876e9ce8..2e5c64d2f 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -1,6 +1,8 @@ //! The query workload: each query, its requirements and its recurrence. use crate::accuracy::AccuracyTarget; +use crate::ir::operator::AtModifier; +use crate::stages::EvaluationGrid; use serde::{Deserialize, Serialize}; use super::data::validate_rate; @@ -222,7 +224,7 @@ impl QueryRecurrence { /// known: an estimated rate, a one-off query without a time, an unknown /// recurrence, or a zero interval. Pass 2's chain widths and the evaluation grids of /// Stage 3's contexts, which - /// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on) + /// [`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on) /// reads, both come from it. pub fn evaluation_grid_ms(&self, range_evaluation: Option) -> Option { let times_grid_ms = match self { @@ -415,6 +417,53 @@ pub struct QueryWorkloadEntry { pub range_evaluation: Option, } +impl QueryWorkloadEntry { + /// The grid, in ms, every evaluation of the query falls on + /// ([`QueryRecurrence::evaluation_grid_ms`] of its range evaluation), + /// the grid of its evaluation context; 1 ms when not known, a grid on + /// which only producers stored at every time deliver. + pub fn evaluation_grid_ms(&self) -> u64 { + self.recurrence + .evaluation_grid_ms(self.range_evaluation) + .unwrap_or(1) + } + + /// The evaluation grid of the query's evaluation context: every + /// [`Self::evaluation_grid_ms`], anchored at the epoch (phase 0). + /// + /// Example: every 20 s at phase 5 s: `{ period_ms: 5_000, phase_ms: 0 }`. + pub fn evaluation_grid(&self) -> EvaluationGrid { + EvaluationGrid { + period_ms: self.evaluation_grid_ms(), + phase_ms: 0, + } + } + + /// How far back from a request's end its evaluations reach, in ms: its + /// range-evaluation span, 0 for one evaluation per run. The output it + /// reads directly is read back that far. + /// + /// Example: a range request over the last hour at 60 s steps: + /// 3,600,000 ms; an instant query: 0. + pub fn range_span_ms(&self) -> u64 { + self.range_evaluation.map_or(0, |range| range.span_ms) + } + + /// The grid, in ms, of the times its requests anchor `@ end()` or + /// `@ start()` (`at`) at: its request end times, shifted back by its + /// range-evaluation span for `@ start()`; 1 ms when not known. + /// + /// Example: a query every 60 s over the last hour: `@ end()` 60,000 + /// ms, `@ start()` gcd(60,000, 3,600,000) = 60,000 ms. + pub fn request_time_grid_ms(&self, at: AtModifier) -> u64 { + let end_ms = self.recurrence.evaluation_grid_ms(None).unwrap_or(1); + match (at, self.range_evaluation) { + (AtModifier::Start, Some(range)) => greatest_common_divisor(end_ms, range.span_ms), + _ => end_ms, + } + } +} + impl From<&BatchEntry> for QueryWorkloadEntry { fn from(entry: &BatchEntry) -> Self { Self { diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 3a16ed13e..4024f5ba1 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -363,7 +363,7 @@ fn a_top_k_ranks_values_of_its_inputs_sign() { /// `@ T` reads a fixed window, and `@ T` over an `@ end()`, `now()` or /// `time()` still depends, since those are not pinned by it. `@ end()` does /// not pin a subtree to a fixed time. A `Scan` reads no time: whether its -/// data arrives is `PhysicalCandidate::delivers_on`'s check. +/// data arrives is `PhysicalASAPMemoCandidates::delivers_on`'s check. /// /// | Group | Expression | Depends | /// |---|---|---| diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index f59c6a3f0..633812520 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -301,10 +301,14 @@ does not allow. With Q0 = `p99 / p50` and Q1 = `p99 / p50 > 1` sharing the ratio, Q0 may read the DDSketch readouts while Q1 reads the ratio and every group below it only exactly; with `quantile_over_time(0.9, m[5m])` asked once at a rank target and once at a relative one, each reads the other's readout -groups only exactly. Stage 3 must enforce the list per query; Pass 2 only -computes it. A scalar root's operands are roots: `scalar(v)` and a scalar -subquery keep the target's metrics, and any other scalar expression reads its -operands only exactly. +groups only exactly. Pass 2 only computes the list; Stage 3 enforces it per +query, in the query's own selection context (the selection MILP's +constraint (X)): no execution on the query's selected path, from its output +down, in a group on its list is a candidate with sketch parameters, while +another query may still read that candidate through its own path. A scalar +root's operands are roots: `scalar(v)` and a scalar subquery keep the +target's metrics, and any other scalar expression reads its operands only +exactly. The list is per group, so it is conservative where one group holds candidates a query may and may not read: `avg by (job)(count_over_time(m[1m]))` diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 0ddf66196..a112954eb 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -479,10 +479,17 @@ then the Pass 2 transformations), and expand shared sizes, tumbling-window horiz (planned) raw windows into concrete groups, so that choosing a threshold is choosing a group; choose one candidate per needed group; compose the choice into one plan and check and cost it with Stage 2 and Stage 3. The search is -the group selection MILP (mixed-integer linear program, #1095): it chooses one -candidate per memo group and execution context, with each shareable candidate -charged once, and is solved once; it prices cost model 1 only (cost models 2 -and 3 are an `UnsupportedCostModel` error, #888). Stage 3 builds the chosen +the selection MILP (mixed-integer linear program, concepts doc +sections 3 and 5) over execution contexts: a candidate with lifetime None or +`OneEvaluation` runs once per context that selects it ($x_{\kappa,v}$; a +context is an evaluation grid shared by the queries on it, an ingestion +pipeline's refresh window, or a materialized candidate's own computation), an +`AcrossEvaluations` candidate once ($x^{\mathrm{mat}}_v$), every edge per +context of its consumer ($y_{\kappa,u,c,i}$, only where `delivers_on` +holds), and a candidate that retains time-indexed rows retains what its +selected consumers read back ($h_v$, constraint (R)). It is solved once and +costs under cost model 1 only (cost models 2 and 3 are an +`UnsupportedCostModel` error). Stage 3 builds the chosen plan, a `PhysicalASAPDAG`, as its last step, so it does not depend on the Stage 2 crate. The facade runs Pass 2, Stage 2 and Stage 3 once, over the full tumbling-window-width domain, with no time limit. @@ -496,7 +503,7 @@ exactly come from `asap_accuracy::ErrorMetricsOfMemo`): the crate root's `generate_logical_asap_memo_candidates` runs them); the memo types Stage 1 hands Stage 2 are in `crates/types/src/stages/` (`asap_types::stages`); the facade that runs the stages is in `crates/planner/src/run.rs`; the search over Stage 2's -candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `groups/`, `milp/`). +candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `selectable.rs`, `cost.rs`, `retention.rs`, `milp/`, `plan.rs`, `report.rs`). | Step | Entry point | Path | |---|---|---| @@ -504,7 +511,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `groups/`, `milp | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{candidates.rs,rollup/,topk.rs,chains.rs}`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`; the readout levels: `asap_accuracy::readout_levels`; the groups each query may read only exactly: `asap_accuracy::ErrorMetricsOfMemo::exact_only_groups_of_query`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | -| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | +| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `retention.rs` for $h_v$ and (R), `exact_only.rs` for each query's exact-only reads $a_{q,e}$ and (X), `dominance.rs` for executions their `OneEvaluation` twin dominates, `equivalent_contexts.rs` for the copies of an equivalent evaluation context, which share the original's variables, `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -1273,11 +1280,9 @@ not check executor support. It then removes the candidates no buildable input or query output can use (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`), repeating until nothing more is removed. Stage 2 makes no execution-scope -copies: how many times a candidate runs is Stage 3's choice, and a -selection names each run as a node of its `DagOfSelectedCandidates`. -Today's Stage 3 engine, whose MILP runs each candidate at most once, makes -the copies it needs itself before solving and decodes each selected copy -into a node. +copies: how many times a candidate runs is Stage 3's choice. Stage 3 runs +a candidate with lifetime None or `OneEvaluation` once per execution +context that selects it, and builds each run as a node of the plan. *Example for Algorithm 3* ([Example 4, Pattern B](#example-4-materialization-of-window-summaries-in-physical-planning)). The logical candidate is the `Tumbling(1m)` KLL option from Algorithm 2. The diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index eb5cea838..3f9b6aac8 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -76,9 +76,9 @@ same pipeline behind the `OptimizationPass` trait. | Pass 2 steps: rollup from the finest common group (shared producers across group keys), top-k reuse, readout requirements, dead-candidate elimination, chain groups (tumbling windows) and the logical edges | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates/src/`): `lib.rs` (the steps in order), `rules/` (`candidates.rs`, `rollup/`, `topk.rs`, `chains.rs`), `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs` and `evaluation_grids.rs`; the groups each query may read only exactly come from `asap_accuracy::ErrorMetricsOfMemo` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | none today (the standalone enumeration was deleted) | | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | -| Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes in `asap-types`, `stages/physical/memo_candidates/mod.rs` | +| Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes, the times a stored result is kept at (`stored_times`), which contexts a candidate delivers on (`delivers_on`) and the candidates as the time walks see them (`TimeIndexedPlan`) in `asap-types`, `stages/physical/memo_candidates/mod.rs` | | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state/mod.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | -| Stage 3: the selectable edges (`selectable.rs`), each candidate's cost $w_v$ and each edge's $c_{u,c,i}$ (`cost.rs`), retention (`retention.rs`), the baseline MILP (`milp/`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | +| Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R), and with each query's exact-only reads $a_{q,e}$ and (X) (`milp/`, `exact_only.rs`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | | The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | | The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 8b3f3a847..3031b3595 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -127,7 +127,7 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- - `Selection.plan` is the selected candidate's roots with the weight requirements Stage 3 confirmed from the data facts. - A producer feeds a consumer only in a context it delivers on - (`PhysicalCandidate::delivers_on`). A result stored across evaluations + (`PhysicalASAPMemoCandidates::delivers_on`). A result stored across evaluations delivers on every context only when it is final, the same at every time it is read. This assumes no row arrives out of order: at evaluation time t, every row with `ts <= t` has arrived and never changes. A result is